diff --git a/translib/common_app.go b/translib/common_app.go index 6f57ddabfa0a..725ab7707c79 100644 --- a/translib/common_app.go +++ b/translib/common_app.go @@ -20,26 +20,32 @@ package translib import ( "errors" - "fmt" "strings" log "github.com/golang/glog" "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ygot/ytypes" + "github.com/openconfig/ygot/util" "reflect" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/Azure/sonic-mgmt-common/translib/transformer" - "encoding/json" + "github.com/Azure/sonic-mgmt-common/translib/utils" + "sync" ) var () type CommonApp struct { pathInfo *PathInfo + body []byte ygotRoot *ygot.GoStruct ygotTarget *interface{} - cmnAppTableMap map[string]map[string]db.Value - cmnAppOrdTbllist []string + skipOrdTableChk bool + cmnAppTableMap map[int]map[db.DBNum]map[string]map[string]db.Value + cmnAppYangDefValMap map[string]map[string]db.Value + cmnAppYangAuxMap map[string]map[string]db.Value + appOptions } var cmnAppInfo = appInfo{appType: reflect.TypeOf(CommonApp{}), @@ -57,13 +63,24 @@ func init() { log.Fatal("Register Common app module with App Interface failed with error=", err, "for path=", mdl_pth) } } - + mdlCpblt := transformer.AddModelCpbltInfo() + if mdlCpblt == nil { + log.Warning("Failure in fetching model capabilities data.") + } else { + for yngMdlNm, mdlDt := range(mdlCpblt) { + err := addModel(&ModelData{Name: yngMdlNm, Org: mdlDt.Org, Ver: mdlDt.Ver}) + if err != nil { + log.Warningf("Adding model data for module %v to appinterface failed with error=%v", yngMdlNm, err) + } + } + } } func (app *CommonApp) initialize(data appData) { log.Info("initialize:path =", data.path) pathInfo := NewPathInfo(data.path) - *app = CommonApp{pathInfo: pathInfo, ygotRoot: data.ygotRoot, ygotTarget: data.ygotTarget} + *app = CommonApp{pathInfo: pathInfo, body: data.payload, ygotRoot: data.ygotRoot, ygotTarget: data.ygotTarget, skipOrdTableChk: false} + app.appOptions = data.appOptions } @@ -112,15 +129,65 @@ func (app *CommonApp) translateGet(dbs [db.MaxDB]*db.DB) error { return err } -func (app *CommonApp) translateAction(dbs [db.MaxDB]*db.DB) error { - err := errors.New("Not supported") - return err +func (app *CommonApp) translateSubscribe(dbs [db.MaxDB]*db.DB, path string) (*notificationOpts, *notificationInfo, error) { + var err error + var subscDt transformer.XfmrTranslateSubscribeInfo + var notifInfo notificationInfo + var notifOpts notificationOpts + txCache := new(sync.Map) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + + log.Info("tranlateSubscribe:path", path) + subscDt, err = transformer.XlateTranslateSubscribe(path, dbs, txCache) + if subscDt.PType == transformer.OnChange { + notifOpts.pType = OnChange + } else { + notifOpts.pType = Sample + } + notifOpts.mInterval = subscDt.MinInterval + notifOpts.isOnChangeSupported = subscDt.OnChange + if err != nil { + log.Infof("returning: notificationOpts - %v, nil, error - %v", notifOpts, err) + return ¬ifOpts, nil, err + } + if subscDt.DbDataMap == nil { + log.Infof("DB data is nil so returning: notificationOpts - %v, nil, error - %v", notifOpts, err) + return ¬ifOpts, nil, err + } else { + for dbNo, dbDt := range(subscDt.DbDataMap) { + if (len(dbDt) == 0) { //ideally all tables for a given uri should be from same DB + continue + } + log.Infof("Adding to notifInfo, Db Data - %v for DB No - %v", dbDt, dbNo) + notifInfo.dbno = dbNo + // in future there will be, multi-table in a DB, support from translib, for now its just single table + for tblNm, tblDt := range(dbDt) { + notifInfo.table = db.TableSpec{Name:tblNm} + if (len(tblDt) == 1) { + for tblKy := range(tblDt) { + notifInfo.key = asKey(tblKy) + notifInfo.needCache = subscDt.NeedCache + } + } else { + if (len(tblDt) > 1) { + log.Warningf("More than one DB key found for subscription path - %v", path) + } else { + log.Warningf("No DB key found for subscription path - %v", path) + } + return ¬ifOpts, nil, err + } + + } + } + } + log.Infof("For path - %v, returning: notifOpts - %v, notifInfo - %v, error - nil", path, notifOpts, notifInfo) + return ¬ifOpts, ¬ifInfo, nil } -func (app *CommonApp) translateSubscribe(dbs [db.MaxDB]*db.DB, path string) (*notificationOpts, *notificationInfo, error) { - err := errors.New("Not supported") - notifInfo := notificationInfo{dbno: db.ConfigDB} - return nil, ¬ifInfo, err +func (app *CommonApp) translateAction(dbs [db.MaxDB]*db.DB) error { + var err error + log.Info("translateAction:path =", app.pathInfo.Path, app.body) + return err } func (app *CommonApp) processCreate(d *db.DB) (SetResponse, error) { @@ -131,7 +198,7 @@ func (app *CommonApp) processCreate(d *db.DB) (SetResponse, error) { targetType := reflect.TypeOf(*app.ygotTarget) log.Infof("processCreate: Target object is a <%s> of Type: %s", targetType.Kind().String(), targetType.Elem().Name()) if err = app.processCommon(d, CREATE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -143,7 +210,7 @@ func (app *CommonApp) processUpdate(d *db.DB) (SetResponse, error) { var resp SetResponse log.Info("processUpdate:path =", app.pathInfo.Path) if err = app.processCommon(d, UPDATE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -155,7 +222,7 @@ func (app *CommonApp) processReplace(d *db.DB) (SetResponse, error) { var resp SetResponse log.Info("processReplace:path =", app.pathInfo.Path) if err = app.processCommon(d, REPLACE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } return resp, err @@ -165,10 +232,10 @@ func (app *CommonApp) processDelete(d *db.DB) (SetResponse, error) { var err error var resp SetResponse - log.Info("processDelete:path =", app.pathInfo.Path) + log.Infof("processDelete:path = %s, deleteEmptyEntry = %v", app.pathInfo.Path, app.deleteEmptyEntry) if err = app.processCommon(d, DELETE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -180,145 +247,292 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) { var payload []byte var resPayload []byte log.Info("processGet:path =", app.pathInfo.Path) + txCache := new(sync.Map) - payload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, app.ygotRoot, dbs) - if err != nil { - log.Error("transformer.transformer.GetAndXlateFromDB failure. error:", err) - return GetResponse{Payload: payload, ErrSrc: AppErr}, err - } + for { + origXfmrYgotRoot, _ := ygot.DeepCopy((*app.ygotRoot).(ygot.GoStruct)) - targetObj, _ := (*app.ygotTarget).(ygot.GoStruct) - if targetObj != nil { - err = ocbinds.Unmarshal(payload, targetObj) + isEmptyPayload := false + appYgotStruct := (*app.ygotRoot).(ygot.GoStruct) + payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache) if err != nil { - log.Error("ocbinds.Unmarshal() failed. error:", err) - return GetResponse{Payload: payload, ErrSrc: AppErr}, err + log.Warning("transformer.GetAndXlateFromDB() returned : ", err) + resPayload = payload + break + } + if strings.HasPrefix(app.pathInfo.Path, "/sonic") && isEmptyPayload { + log.Info("transformer.GetAndXlateFromDB() returned EmptyPayload") + resPayload = payload + break } - resPayload, err = generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) - if err != nil { - log.Error("generateGetResponsePayload() failed") - return GetResponse{Payload: payload, ErrSrc: AppErr}, err + targetObj, tgtObjCastOk := (*app.ygotTarget).(ygot.GoStruct) + if !tgtObjCastOk { + /*For ygotTarget populated by tranlib, for query on leaf level and list(without instance) level, + casting to GoStruct fails so use the parent node of ygotTarget to Unmarshall the payload into*/ + log.Infof("Use GetParentNode() instead of casting ygotTarget to GoStruct, uri - %v", app.pathInfo.Path) + targetUri := app.pathInfo.Path + parentTargetObj, _, getParentNodeErr := getParentNode(&targetUri, (*app.ygotRoot).(*ocbinds.Device)) + if getParentNodeErr != nil { + log.Warningf("getParentNode() failure for uri %v", app.pathInfo.Path) + resPayload = payload + break + } + if parentTargetObj != nil { + targetObj, tgtObjCastOk = (*parentTargetObj).(ygot.GoStruct) + if !tgtObjCastOk { + log.Warningf("Casting of parent object returned from getParentNode() to GoStruct failed(uri - %v)", app.pathInfo.Path) + resPayload = payload + break + } + } else { + log.Warningf("getParentNode() returned a nil Object for uri %v", app.pathInfo.Path) + resPayload = payload + break + } + } + if targetObj != nil { + updateListEntriesOpt := ytypes.AllowUpdateInListMap{} + err = ocbinds.Unmarshal(payload, targetObj, &updateListEntriesOpt) + if err != nil { + log.Warning("ocbinds.Unmarshal() returned : ", err) + resPayload = payload + break + } + + resYgot := (*app.ygotRoot) + if !strings.HasPrefix(app.pathInfo.Path, "/sonic") { + if isEmptyPayload { + if areEqual(appYgotStruct, origXfmrYgotRoot) { + log.Info("origXfmrYgotRoot and appYgotStruct are equal.") + // No data available in appYgotStruct. + if transformer.IsLeafNode(app.pathInfo.Path) { + //if leaf not exist in DB subtree won't fill ygotRoot, as per RFC return err + resPayload = payload + log.Info("No data found for leaf.") + err = tlerr.NotFound("Resource not found") + break + } + resPayload = payload + log.Info("No data available") + //TODO: Return not found error + //err = tlerr.NotFound("Resource not found") + break + + } + resYgot = appYgotStruct + } + } + if resYgot != nil { + resPayload, err = generateGetResponsePayload(app.pathInfo.Path, resYgot.(*ocbinds.Device), app.ygotTarget) + if err != nil { + log.Warning("generateGetResponsePayload() couldn't generate payload.") + resPayload = payload + } + } else { + resPayload = payload + } + + break + } else { + log.Warning("processGet. targetObj is null. Unable to Unmarshal payload") + resPayload = payload + break } - var dat map[string]interface{} - err = json.Unmarshal(resPayload, &dat) - } else { - log.Warning("processGet. targetObj is null. Unable to Unmarshal payload") - resPayload = payload } return GetResponse{Payload: resPayload}, err } func (app *CommonApp) processAction(dbs [db.MaxDB]*db.DB) (ActionResponse, error) { - var resp ActionResponse - err := errors.New("Not implemented") + var resp ActionResponse + var err error + + resp.Payload, err = transformer.CallRpcMethod(app.pathInfo.Path, app.body, dbs) + log.Info("transformer.CallRpcMethod() returned") - return resp, err + return resp, err } func (app *CommonApp) translateCRUDCommon(d *db.DB, opcode int) ([]db.WatchKeys, error) { var err error var keys []db.WatchKeys var tblsToWatch []*db.TableSpec - var OrdTblList []string - var moduleNm string + txCache := new(sync.Map) log.Info("translateCRUDCommon:path =", app.pathInfo.Path) - /* retrieve schema table order for incoming module name request */ - moduleNm, err = transformer.GetModuleNmFromPath(app.pathInfo.Path) - if (err != nil) || (len(moduleNm) == 0) { - log.Error("GetModuleNmFromPath() failed") - return keys, err - } - log.Info("getModuleNmFromPath() returned module name = ", moduleNm) - OrdTblList, err = transformer.GetOrdDBTblList(moduleNm) - if (err != nil) || (len(OrdTblList) == 0) { - log.Error("GetOrdDBTblList() failed") - return keys, err - } - - log.Info("GetOrdDBTblList() returned ordered table list = ", OrdTblList) - app.cmnAppOrdTbllist = OrdTblList - - /* enhance this to handle dependent tables - need CVL to provide list of such tables for a given request */ - for _, tblnm := range OrdTblList { // OrdTblList already has has all tables corresponding to a module - tblsToWatch = append(tblsToWatch, &db.TableSpec{Name: tblnm}) - } - log.Info("Tables to watch", tblsToWatch) - - cmnAppInfo.tablesToWatch = tblsToWatch - // translate yang to db - result, err := transformer.XlateToDb(app.pathInfo.Path, opcode, d, (*app).ygotRoot, (*app).ygotTarget) - fmt.Println(result) - log.Info("transformer.XlateToDb() returned", result) + result, defValMap, auxMap, err := transformer.XlateToDb(app.pathInfo.Path, opcode, d, (*app).ygotRoot, (*app).ygotTarget, (*app).body, txCache, &app.skipOrdTableChk) + log.Info("transformer.XlateToDb() returned result DB map - ", result, "\nDefault value Db Map - ", defValMap, "\nAux Db Map - ", auxMap) + if err != nil { - log.Error(err) + log.Warning(err) return keys, err } + app.cmnAppTableMap = result + app.cmnAppYangDefValMap = defValMap + app.cmnAppYangAuxMap = auxMap //used for Replace case if len(result) == 0 { - log.Error("XlatetoDB() returned empty map") - err = errors.New("transformer.XlatetoDB() returned empty map") + log.Info("XlatetoDB() returned empty map") + //Note: Get around for no redis ABNF Schema for set(temporary) + //`err = errors.New("transformer.XlatetoDB() returned empty map") return keys, err } - app.cmnAppTableMap = result - keys, err = app.generateDbWatchKeys(d, false) + moduleNm, err := transformer.GetModuleNmFromPath(app.pathInfo.Path) + if (err != nil) || (len(moduleNm) == 0) { + log.Warning("GetModuleNmFromPath() couldn't fetch module name.") + return keys, err + } + + var resultTblList []string + for _, dbMap := range result { //Get dependency list for all tables in result + for _, resMap := range dbMap { //Get dependency list for all tables in result + for tblnm := range resMap { //Get dependency list for all tables in result + resultTblList = append(resultTblList, tblnm) + } + } + } + log.Info("Result Tables List", resultTblList) + + // Get list of tables to watch + if len(resultTblList) > 0 { + depTbls := transformer.GetTablesToWatch(resultTblList, moduleNm) + if len(depTbls) == 0 { + log.Warningf("Couldn't get Tables to watch for module %v", moduleNm) + err = errors.New("GetTablesToWatch returned empty slice") + return keys, err + } + for _, tbl := range depTbls { + tblsToWatch = append(tblsToWatch, &db.TableSpec{Name: tbl}) + } + } + log.Info("Tables to watch", tblsToWatch) + cmnAppInfo.tablesToWatch = tblsToWatch + keys, err = app.generateDbWatchKeys(d, false) return keys, err } func (app *CommonApp) processCommon(d *db.DB, opcode int) error { var err error + if len(app.cmnAppTableMap) == 0 { + return err + } log.Info("Processing DB operation for ", app.cmnAppTableMap) switch opcode { case CREATE: log.Info("CREATE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case UPDATE: log.Info("UPDATE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case REPLACE: log.Info("REPLACE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case DELETE: log.Info("DELETE case") - err = app.cmnAppDelDbOpn(d, opcode) } - if err != nil { - log.Info("Returning from processCommon() - fail") - } else { - log.Info("Returning from processCommon() - success") + + // Handle delete first if any available + if _, ok := app.cmnAppTableMap[DELETE][db.ConfigDB]; ok { + err = app.cmnAppDelDbOpn(d, DELETE, app.cmnAppTableMap[DELETE][db.ConfigDB]) + if err != nil { + log.Info("Process delete fail. cmnAppDelDbOpn error:", err) + return err + } + } + // Handle create operation next + if _, ok := app.cmnAppTableMap[CREATE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, CREATE, app.cmnAppTableMap[CREATE][db.ConfigDB]) + if err != nil { + log.Info("Process create fail. cmnAppCRUCommonDbOpn error:", err) + return err + } + } + // Handle update and replace operation next + if _, ok := app.cmnAppTableMap[UPDATE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, UPDATE, app.cmnAppTableMap[UPDATE][db.ConfigDB]) + if err != nil { + log.Info("Process update fail. cmnAppCRUCommonDbOpn error:", err) + return err + } + } + if _, ok := app.cmnAppTableMap[REPLACE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, REPLACE, app.cmnAppTableMap[REPLACE][db.ConfigDB]) + if err != nil { + log.Info("Process replace fail. cmnAppCRUCommonDbOpn error:", err) + return err + } } + log.Info("Returning from processCommon() - success") return err } -func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { +func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int, dbMap map[string]map[string]db.Value) error { var err error var cmnAppTs *db.TableSpec + var xfmrTblLst []string + var resultTblLst []string - /* currently ordered by schema table order needs to be discussed */ - for _, tblNm := range app.cmnAppOrdTbllist { + for tblNm := range(dbMap) { + xfmrTblLst = append(xfmrTblLst, tblNm) + } + resultTblLst, err = utils.SortAsPerTblDeps(xfmrTblLst) + if err != nil { + return err + } + + /* CVL sorted order is in child first, parent later order. CRU ops from parent first order */ + for idx := len(resultTblLst)-1; idx >= 0; idx-- { + tblNm := resultTblLst[idx] log.Info("In Yang to DB map returned from transformer looking for table = ", tblNm) - if tblVal, ok := app.cmnAppTableMap[tblNm]; ok { + if tblVal, ok := dbMap[tblNm]; ok { cmnAppTs = &db.TableSpec{Name: tblNm} log.Info("Found table entry in yang to DB map") + if ((tblVal == nil) || (len(tblVal) == 0)) { + log.Info("No table instances/rows found.") + continue + } for tblKey, tblRw := range tblVal { - log.Info("Processing Table key and row ", tblKey, tblRw) + log.Info("Processing Table key ", tblKey) + // REDIS doesn't allow to create a table instance without any fields + if tblRw.Field == nil { + tblRw.Field = map[string]string{"NULL": "NULL"} + } + if len(tblRw.Field) == 0 { + tblRw.Field["NULL"] = "NULL" + } + if len(tblRw.Field) > 1 { + delete(tblRw.Field, "NULL") + } existingEntry, _ := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) switch opcode { case CREATE: if existingEntry.IsPopulated() { - log.Info("Entry already exists hence return.") - return tlerr.AlreadyExists("Entry %s already exists", tblKey) + log.Info("Create case - Entry ", tblKey, " already exists hence modifying it.") + /* Handle leaf-list merge if any leaf-list exists + A leaf-list field in redis has "@" suffix as per swsssdk convention. + */ + resTblRw := db.Value{Field: map[string]string{}} + resTblRw = checkAndProcessLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) + log.Info("Processing Table row ", resTblRw) + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) + if err != nil { + log.Warning("CREATE case - d.ModEntry() failure") + return err + } } else { + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("Entry ", tblKey, " doesn't exist so fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("CREATE case - d.CreateEntry() failure") + log.Warning("CREATE case - d.CreateEntry() failure") return err } } @@ -326,37 +540,82 @@ func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { if existingEntry.IsPopulated() { log.Info("Entry already exists hence modifying it.") /* Handle leaf-list merge if any leaf-list exists - A leaf-list field in redis has "@" suffix as per swsssdk convention. - */ + A leaf-list field in redis has "@" suffix as per swsssdk convention. + */ resTblRw := db.Value{Field: map[string]string{}} resTblRw = checkAndProcessLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) if err != nil { - log.Error("UPDATE case - d.ModEntry() failure") + log.Warning("UPDATE case - d.ModEntry() failure") return err } } else { - // workaround to patch operation from CLI - log.Info("Create(pathc) an entry.") - err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + // workaround to patch operation from CLI + log.Info("Create(pathc) an entry.") + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("Entry ", tblKey, " doesn't exist so fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) + err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("UPDATE case - d.CreateEntry() failure") + log.Warning("UPDATE case - d.CreateEntry() failure") return err } } case REPLACE: + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("For entry ", tblKey, ", being replaced, fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) if existingEntry.IsPopulated() { - log.Info("Entry already exists hence execute db.SetEntry") - err := d.SetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) - if err != nil { - log.Error("REPLACE case - d.SetEntry() failure") - return err + log.Info("Entry already exists.") + auxRwOk := false + auxRw := db.Value{Field: map[string]string{}} + auxRw, auxRwOk = app.cmnAppYangAuxMap[tblNm][tblKey] + log.Info("Process Aux row ", auxRw) + isTlNd := false + if !strings.HasPrefix(app.pathInfo.Path, "/sonic") { + isTlNd, err = transformer.IsTerminalNode(app.pathInfo.Path) + log.Info("transformer.IsTerminalNode() returned - ", isTlNd, " error ", err) + if err != nil { + return err + } + } + if isTlNd && isPartialReplace(existingEntry, tblRw, auxRw) { + log.Info("Since its partial replace modifying fields - ", tblRw) + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + if err != nil { + log.Warning("REPLACE case - d.ModEntry() failure") + return err + } + if auxRwOk { + if len(auxRw.Field) > 0 { + log.Info("Since its partial replace delete aux fields - ", auxRw) + err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, auxRw) + if err != nil { + log.Warning("REPLACE case - d.DeleteEntryFields() failure") + return err + } + } + } + } else { + err := d.SetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + if err != nil { + log.Warning("REPLACE case - d.SetEntry() failure") + return err + } } } else { log.Info("Entry doesn't exist hence create it.") err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("REPLACE case - d.CreateEntry() failure") + log.Warning("REPLACE case - d.CreateEntry() failure") return err } } @@ -367,26 +626,64 @@ func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { return err } -func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { +func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int, dbMap map[string]map[string]db.Value) error { var err error var cmnAppTs, dbTblSpec *db.TableSpec + var moduleNm string + var xfmrTblLst []string + var resultTblLst []string + var ordTblList []string - /* needs enhancements from CVL to give table dependencies, and grouping of related tables only - if such a case where the sonic yang has unrelated tables */ - for tblidx, tblNm := range app.cmnAppOrdTbllist { + for tblNm := range(dbMap) { + xfmrTblLst = append(xfmrTblLst, tblNm) + } + resultTblLst, err = utils.SortAsPerTblDeps(xfmrTblLst) + if err != nil { + return err + } + + + /* Retrieve module Name */ + moduleNm, err = transformer.GetModuleNmFromPath(app.pathInfo.Path) + if (err != nil) || (len(moduleNm) == 0) { + log.Warning("GetModuleNmFromPath() failed") + return err + } + log.Info("getModuleNmFromPath() returned module name = ", moduleNm) + + /* resultTblLst has child first, parent later order */ + for _, tblNm := range resultTblLst { log.Info("In Yang to DB map returned from transformer looking for table = ", tblNm) - if tblVal, ok := app.cmnAppTableMap[tblNm]; ok { + if tblVal, ok := dbMap[tblNm]; ok { cmnAppTs = &db.TableSpec{Name: tblNm} log.Info("Found table entry in yang to DB map") + if !app.skipOrdTableChk { + ordTblList = transformer.GetXfmrOrdTblList(tblNm) + if len(ordTblList) == 0 { + ordTblList = transformer.GetOrdTblList(tblNm, moduleNm) + } + if len(ordTblList) == 0 { + log.Warning("GetOrdTblList returned empty slice") + err = errors.New("GetOrdTblList returned empty slice. Insufficient information to process request") + return err + } + log.Infof("GetOrdTblList for table - %v, module %v returns %v", tblNm, moduleNm, ordTblList) + } if len(tblVal) == 0 { log.Info("DELETE case - No table instances/rows found hence delete entire table = ", tblNm) - for idx := len(app.cmnAppOrdTbllist)-1; idx >= tblidx+1; idx-- { - log.Info("Since parent table is to be deleted, first deleting child table = ", app.cmnAppOrdTbllist[idx]) - dbTblSpec = &db.TableSpec{Name: app.cmnAppOrdTbllist[idx]} - err = d.DeleteTable(dbTblSpec) - if err != nil { - log.Warning("DELETE case - d.DeleteTable() failure for Table = ", app.cmnAppOrdTbllist[idx]) - return err + if !app.skipOrdTableChk { + for _, ordtbl := range ordTblList { + if ordtbl == tblNm { + // Handle the child tables only till you reach the parent table entry + break + } + log.Info("Since parent table is to be deleted, first deleting child table = ", ordtbl) + dbTblSpec = &db.TableSpec{Name: ordtbl} + err = d.DeleteTable(dbTblSpec) + if err != nil { + log.Warning("DELETE case - d.DeleteTable() failure for Table = ", ordtbl) + return err + } } } err = d.DeleteTable(cmnAppTs) @@ -395,8 +692,8 @@ func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { return err } log.Info("DELETE case - Deleted entire table = ", tblNm) - log.Info("Done processing all tables.") - break + // Continue to repeat ordered deletion for all tables + continue } @@ -404,40 +701,67 @@ func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { if len(tblRw.Field) == 0 { log.Info("DELETE case - no fields/cols to delete hence delete the entire row.") log.Info("First, delete child table instances that correspond to parent table instance to be deleted = ", tblKey) - for idx := len(app.cmnAppOrdTbllist)-1; idx >= tblidx+1; idx-- { - dbTblSpec = &db.TableSpec{Name: app.cmnAppOrdTbllist[idx]} - keyPattern := tblKey + "|*" - log.Info("Key pattern to be matched for deletion = ", keyPattern) - err = d.DeleteKeys(dbTblSpec, db.Key{Comp: []string{keyPattern}}) - if err != nil { - log.Warning("DELETE case - d.DeleteTable() failure for Table = ", app.cmnAppOrdTbllist[idx]) - return err + if !app.skipOrdTableChk { + for _, ordtbl := range ordTblList { + if ordtbl == tblNm { + // Handle the child tables only till you reach the parent table entry + break; + } + dbTblSpec = &db.TableSpec{Name: ordtbl} + keyPattern := tblKey + "|*" + log.Info("Key pattern to be matched for deletion = ", keyPattern) + err = d.DeleteKeys(dbTblSpec, db.Key{Comp: []string{keyPattern}}) + if err != nil { + log.Warning("DELETE case - d.DeleteTable() failure for Table = ", ordtbl) + return err + } + log.Info("Deleted keys matching parent table key pattern for child table = ", ordtbl) } - log.Info("Deleted keys matching parent table key pattern for child table = ", app.cmnAppOrdTbllist[idx]) - } err = d.DeleteEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) - if err != nil { - log.Warning("DELETE case - d.DeleteEntry() failure") - return err - } + if err != nil { + log.Warning("DELETE case - d.DeleteEntry() failure") + return err + } log.Info("Finally deleted the parent table row with key = ", tblKey) } else { log.Info("DELETE case - fields/cols to delete hence delete only those fields.") - existingEntry, _ := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) - if !existingEntry.IsPopulated() { + existingEntry, exstErr := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) + if exstErr != nil { log.Info("Table Entry from which the fields are to be deleted does not exist") + err = exstErr return err } /* handle leaf-list merge if any leaf-list exists */ resTblRw := checkAndProcessLeafList(existingEntry, tblRw, DELETE, d, tblNm, tblKey) - err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) - if err != nil { - log.Error("DELETE case - d.DeleteEntryFields() failure") - return err + log.Info("DELETE case - checkAndProcessLeafList() returned table row ", resTblRw) + if len(resTblRw.Field) > 0 { + if !app.deleteEmptyEntry { + /* add the NULL field if the last field gets deleted && deleteEmpyEntry is false */ + deleteCount := 0 + for field := range existingEntry.Field { + if resTblRw.Has(field) { + deleteCount++ + } + } + if deleteCount == len(existingEntry.Field) { + nullTblRw := db.Value{Field: map[string]string{"NULL": "NULL"}} + log.Info("Last field gets deleted, add NULL field to keep an db entry") + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, nullTblRw) + if err != nil { + log.Warning("UPDATE case - d.ModEntry() failure") + return err + } + } + } + /* deleted fields */ + err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) + if err != nil { + log.Warning("DELETE case - d.DeleteEntryFields() failure") + return err + } } } - } } } /* end of ordered table list for loop */ @@ -458,8 +782,14 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, for field, value := range tblRw.Field { if strings.HasSuffix(field, "@") { exstLst := existingEntry.GetList(field) + log.Infof("Existing DB value for field %v - %v", field, exstLst) + var valueLst []string + if value != "" { //zero len string as leaf-list value is treated as delete entire leaf-list + valueLst = strings.Split(value, ",") + } + log.Infof("Incoming value for field %v - %v", field, valueLst) if len(exstLst) != 0 { - valueLst := strings.Split(value, ",") + log.Infof("Existing list is not empty for field %v", field) for _, item := range valueLst { if !contains(exstLst, item) { if opcode == UPDATE { @@ -467,23 +797,48 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, } } else { if opcode == DELETE { - exstLst = removeElement(exstLst, item) + exstLst = utils.RemoveElement(exstLst, item) } } } - log.Infof("For field %v value after merge %v", field, exstLst) + log.Infof("For field %v value after merging incoming with existing %v", field, exstLst) if opcode == DELETE { - mergeTblRw.SetList(field, exstLst) - delete(tblRw.Field, field) + if len(valueLst) > 0 { + mergeTblRw.SetList(field, exstLst) + if len(exstLst) == 0 { + tblRw.Field[field] = "" + } else { + delete(tblRw.Field, field) + } + } + } else if opcode == UPDATE { + tblRw.SetList(field, exstLst) } - } - tblRw.SetList(field, exstLst) + } else { //when existing list is empty(either empty string val in field or no field at all n entry) + log.Infof("Existing list is empty for field %v", field) + if opcode == UPDATE { + if len(valueLst) > 0 { + exstLst = valueLst + tblRw.SetList(field, exstLst) + } else { + tblRw.Field[field] = "" + } + } else if opcode == DELETE { + _, fldExistsOk := existingEntry.Field[field] + if (fldExistsOk && (len(valueLst) == 0)) { + tblRw.Field[field] = "" + } else { + delete(tblRw.Field, field) + } + } + } } } /* delete specific item from leaf-list */ if opcode == DELETE { if len(mergeTblRw.Field) == 0 { + log.Infof("mergeTblRow is empty - Returning Table Row %v", tblRw) return tblRw } err := d.ModEntry(dbTblSpec, db.Key{Comp: []string{tblKey}}, mergeTblRw) @@ -495,3 +850,46 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, return tblRw } +// This function is a copy of the function areEqual in ygot.util package. +// areEqual compares a and b. If a and b are both pointers, it compares the +// values they are pointing to. +func areEqual(a, b interface{}) bool { + if util.IsValueNil(a) && util.IsValueNil(b) { + return true + } + va, vb := reflect.ValueOf(a), reflect.ValueOf(b) + if va.Kind() == reflect.Ptr && vb.Kind() == reflect.Ptr { + return reflect.DeepEqual(va.Elem().Interface(), vb.Elem().Interface()) + } + + return reflect.DeepEqual(a, b) +} + +func isPartialReplace(exstRw db.Value, replTblRw db.Value, auxRw db.Value) bool { + /* if existing entry contains field thats not present in result, + default and auxillary map then its a partial replace + */ + partialReplace := false + for exstFld := range exstRw.Field { + if exstFld == "NULL" { + continue + } + isIncomingFld := false + if replTblRw.Has(exstFld) { + continue + } + if auxRw.Has(exstFld) { + continue + } + if !isIncomingFld { + log.Info("Entry contains field ", exstFld, " not found in result, default and aux fields hence its partial replace.") + partialReplace = true + break + } + } + log.Info("returning partialReplace - ", partialReplace) + return partialReplace +} + + + diff --git a/translib/tlerr/tlerr.go b/translib/tlerr/tlerr.go index 350a8602166b..93682576515d 100644 --- a/translib/tlerr/tlerr.go +++ b/translib/tlerr/tlerr.go @@ -103,12 +103,19 @@ func (e TranslibSyntaxValidationError) Error() string { } type TranslibUnsupportedClientVersion struct { - ClientVersion string - ServerVersion string - ServerBaseVersion string + ClientVersion string + ServerVersion string + ServerBaseVersion string } func (e TranslibUnsupportedClientVersion) Error() string { - return p.Sprintf("Unsupported client version %s", e.ClientVersion) + return p.Sprintf("Unsupported client version %s", e.ClientVersion) } +type TranslibXfmrRetError struct { + XlateFailDelReq bool +} + +func (e TranslibXfmrRetError) Error() string { + return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq) +} diff --git a/translib/transformer/transformer.go b/translib/transformer/transformer.go index 2a06c2dde68d..cca5dc52556c 100644 --- a/translib/transformer/transformer.go +++ b/translib/transformer/transformer.go @@ -21,7 +21,6 @@ package transformer import ( "fmt" "github.com/openconfig/goyang/pkg/yang" - "github.com/openconfig/ygot/ygot" "os" "strings" "bufio" @@ -30,15 +29,8 @@ import ( ) var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models path - -var entries = map[string]*yang.Entry{} - -//Interface for xfmr methods -type xfmrInterface interface { - tableXfmr(s *ygot.GoStruct, t *interface{}) (string, error) - keyXfmr(s *ygot.GoStruct, t *interface{}) (string, error) - fieldXfmr(s *ygot.GoStruct, t *interface{}) (string, error) -} +var ModelsListFile = "models_list" +var TblInfoJsonFile = "sonic_table_info.json" func reportIfError(errs []error) { if len(errs) > 0 { @@ -50,7 +42,7 @@ func reportIfError(errs []error) { func getOcModelsList () ([]string) { var fileList []string - file, err := os.Open(YangPath + "models_list") + file, err := os.Open(YangPath + ModelsListFile) if err != nil { return fileList } @@ -58,7 +50,7 @@ func getOcModelsList () ([]string) { scanner := bufio.NewScanner(file) for scanner.Scan() { fileEntry := scanner.Text() - if strings.HasPrefix(fileEntry, "#") != true { + if !strings.HasPrefix(fileEntry, "#") { _, err := os.Stat(YangPath + fileEntry) if err != nil { continue @@ -86,9 +78,9 @@ func getDefaultModelsList () ([]string) { func init() { initYangModelsPath() - yangFiles := []string{} + initRegex() ocList := getOcModelsList() - yangFiles = getDefaultModelsList() + yangFiles := getDefaultModelsList() yangFiles = append(yangFiles, ocList...) fmt.Println("Yang model List:", yangFiles) err := loadYangModules(yangFiles...) @@ -105,7 +97,7 @@ func initYangModelsPath() { YangPath = path } - fmt.Println("Yang models path:", YangPath) + fmt.Println("Yang modles path:", YangPath) } func loadYangModules(files ...string) error { @@ -147,15 +139,19 @@ func loadYangModules(files ...string) error { } } - sonic_entries := make([]*yang.Entry, len(names)) + sonic_entries := make([]*yang.Entry, 0) oc_entries := make(map[string]*yang.Entry) - oc_annot_entries := make([]*yang.Entry, len(names)) - sonic_annot_entries := make([]*yang.Entry, len(names)) + oc_annot_entries := make([]*yang.Entry, 0) + sonic_annot_entries := make([]*yang.Entry, 0) for _, n := range names { if strings.Contains(n, "annot") && strings.Contains(n, "sonic") { sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n])) } else if strings.Contains(n, "annot") { + yangMdlNmDt := strings.Split(n, "-annot") + if len(yangMdlNmDt) > 0 { + addMdlCpbltEntry(yangMdlNmDt[0]) + } oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n])) } else if strings.Contains(n, "sonic") { sonic_entries = append(sonic_entries, yang.ToEntry(mods[n])) @@ -164,6 +160,37 @@ func loadYangModules(files ...string) error { } } + // populate model capabilities data + for yngMdlNm := range(xMdlCpbltMap) { + org := "" + ver := "" + ocVerSet := false + yngEntry := oc_entries[yngMdlNm] + if (yngEntry != nil) { + // OC yang has version in standard extension oc-ext:openconfig-version + if strings.HasPrefix(yngMdlNm, "openconfig-") { + for _, ext := range yngEntry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + if tagType == "openconfig-version" { + ver = ext.NName() + fmt.Printf("Found version %v for yang module %v", ver, yngMdlNm) + if len(strings.TrimSpace(ver)) > 0 { + ocVerSet = true + } + break + } + + } + } + } + if ((strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet)) { + // as per RFC7895 revision date to be used as version + ver = mods[yngMdlNm].Current() //gives the most recent revision date for yang module + } + org = mods[yngMdlNm].Organization.Name + addMdlCpbltData(yngMdlNm, ver, org) + } dbMapBuild(sonic_entries) annotDbSpecMap(sonic_annot_entries) annotToDbMapBuild(oc_annot_entries) diff --git a/translib/transformer/xconst.go b/translib/transformer/xconst.go index 8fcdabdc758b..a29c8e907025 100644 --- a/translib/transformer/xconst.go +++ b/translib/transformer/xconst.go @@ -24,6 +24,8 @@ const ( YANG_CONTAINER = "container" YANG_LEAF = "leaf" YANG_LEAF_LIST = "leaf-list" + YANG_CHOICE = "choice" + YANG_CASE = "case" YANG_ANNOT_DB_NAME = "db-name" YANG_ANNOT_TABLE_NAME = "table-name" @@ -46,6 +48,43 @@ const ( XPATH_SEP_FWD_SLASH = "/" XFMR_EMPTY_STRING = "" + XFMR_NONE_STRING = "NONE" SONIC_TABLE_INDEX = 2 + SONIC_LIST_INDEX = 3 + SONIC_FIELD_INDEX = 4 + SONIC_MDL_PFX = "sonic" + OC_MDL_PFX = "openconfig-" + IETF_MDL_PFX = "ietf-" + IANA_MDL_PFX = "iana-" + YTDB_KEY_XFMR_RET_ARGS = 2 + YTDB_KEY_XFMR_RET_VAL_INDX = 0 + YTDB_KEY_XFMR_RET_ERR_INDX = 1 + YTDB_SBT_XFMR_RET_ARGS = 2 + YTDB_SBT_XFMR_RET_VAL_INDX = 0 + YTDB_SBT_XFMR_RET_ERR_INDX = 1 + YTDB_FLD_XFMR_RET_ARGS = 2 + YTDB_FLD_XFMR_RET_VAL_INDX = 0 + YTDB_FLD_XFMR_RET_ERR_INDX = 1 + DBTY_KEY_XFMR_RET_ARGS = 2 + DBTY_KEY_XFMR_RET_VAL_INDX = 0 + DBTY_KEY_XFMR_RET_ERR_INDX = 1 + DBTY_FLD_XFMR_RET_ARGS = 2 + DBTY_FLD_XFMR_RET_VAL_INDX = 0 + DBTY_FLD_XFMR_RET_ERR_INDX = 1 + SUBSC_SBT_XFMR_RET_ARGS = 2 + SUBSC_SBT_XFMR_RET_VAL_INDX = 0 + SUBSC_SBT_XFMR_RET_ERR_INDX = 1 + DBTY_SBT_XFMR_RET_ERR_INDX = 0 + TBL_XFMR_RET_ARGS = 2 + TBL_XFMR_RET_VAL_INDX = 0 + TBL_XFMR_RET_ERR_INDX = 1 + POST_XFMR_RET_ARGS = 2 + POST_XFMR_RET_VAL_INDX = 0 + POST_XFMR_RET_ERR_INDX = 1 + PRE_XFMR_RET_ARGS = 1 + PRE_XFMR_RET_ERR_INDX = 0 + XFMR_INVALID = -1 + XFMR_DISABLE = 0 + XFMR_ENABLE = 1 ) diff --git a/translib/transformer/xfmr_acl.go b/translib/transformer/xfmr_acl.go deleted file mode 100644 index 7db9751537c4..000000000000 --- a/translib/transformer/xfmr_acl.go +++ /dev/null @@ -1,976 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// // -// Copyright 2019 Dell, Inc. // -// // -// Licensed under the Apache License, Version 2.0 (the "License"); // -// you may not use this file except in compliance with the License. // -// You may obtain a copy of the License at // -// // -// http://www.apache.org/licenses/LICENSE-2.0 // -// // -// Unless required by applicable law or agreed to in writing, software // -// distributed under the License is distributed on an "AS IS" BASIS, // -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // -// See the License for the specific language governing permissions and // -// limitations under the License. // -// // -//////////////////////////////////////////////////////////////////////////////// - -package transformer - -import ( - "bytes" - "errors" - "fmt" - log "github.com/golang/glog" - "github.com/openconfig/ygot/ygot" - "reflect" - "strconv" - "strings" - "github.com/Azure/sonic-mgmt-common/translib/db" - "github.com/Azure/sonic-mgmt-common/translib/ocbinds" - "github.com/Azure/sonic-mgmt-common/translib/tlerr" -) - -func init() { - XlateFuncBind("DbToYang_acl_set_name_xfmr", DbToYang_acl_set_name_xfmr) - XlateFuncBind("YangToDb_acl_type_field_xfmr", YangToDb_acl_type_field_xfmr) - XlateFuncBind("DbToYang_acl_type_field_xfmr", DbToYang_acl_type_field_xfmr) - XlateFuncBind("YangToDb_acl_set_key_xfmr", YangToDb_acl_set_key_xfmr) - XlateFuncBind("DbToYang_acl_set_key_xfmr", DbToYang_acl_set_key_xfmr) - XlateFuncBind("YangToDb_acl_entry_key_xfmr", YangToDb_acl_entry_key_xfmr) - XlateFuncBind("DbToYang_acl_entry_key_xfmr", DbToYang_acl_entry_key_xfmr) - XlateFuncBind("DbToYang_acl_entry_sequenceid_xfmr", DbToYang_acl_entry_sequenceid_xfmr) - XlateFuncBind("YangToDb_acl_l2_ethertype_xfmr", YangToDb_acl_l2_ethertype_xfmr) - XlateFuncBind("DbToYang_acl_l2_ethertype_xfmr", DbToYang_acl_l2_ethertype_xfmr) - XlateFuncBind("YangToDb_acl_ip_protocol_xfmr", YangToDb_acl_ip_protocol_xfmr) - XlateFuncBind("DbToYang_acl_ip_protocol_xfmr", DbToYang_acl_ip_protocol_xfmr) - XlateFuncBind("YangToDb_acl_source_port_xfmr", YangToDb_acl_source_port_xfmr) - XlateFuncBind("DbToYang_acl_source_port_xfmr", DbToYang_acl_source_port_xfmr) - XlateFuncBind("YangToDb_acl_destination_port_xfmr", YangToDb_acl_destination_port_xfmr) - XlateFuncBind("DbToYang_acl_destination_port_xfmr", DbToYang_acl_destination_port_xfmr) - XlateFuncBind("YangToDb_acl_tcp_flags_xfmr", YangToDb_acl_tcp_flags_xfmr) - XlateFuncBind("DbToYang_acl_tcp_flags_xfmr", DbToYang_acl_tcp_flags_xfmr) - XlateFuncBind("YangToDb_acl_port_bindings_xfmr", YangToDb_acl_port_bindings_xfmr) - XlateFuncBind("DbToYang_acl_port_bindings_xfmr", DbToYang_acl_port_bindings_xfmr) - XlateFuncBind("YangToDb_acl_forwarding_action_xfmr", YangToDb_acl_forwarding_action_xfmr) - XlateFuncBind("DbToYang_acl_forwarding_action_xfmr", DbToYang_acl_forwarding_action_xfmr) - XlateFuncBind("validate_ipv4", validate_ipv4) - XlateFuncBind("validate_ipv6", validate_ipv6) - XlateFuncBind("acl_post_xfmr", acl_post_xfmr) -} - -const ( - ACL_TABLE = "ACL_TABLE" - RULE_TABLE = "ACL_RULE" - SONIC_ACL_TYPE_IPV4 = "L3" - SONIC_ACL_TYPE_L2 = "L2" - SONIC_ACL_TYPE_IPV6 = "L3V6" - OPENCONFIG_ACL_TYPE_IPV4 = "ACL_IPV4" - OPENCONFIG_ACL_TYPE_IPV6 = "ACL_IPV6" - OPENCONFIG_ACL_TYPE_L2 = "ACL_L2" - ACL_TYPE = "type" - MIN_PRIORITY = 1 - MAX_PRIORITY = 65535 -) - -/* E_OpenconfigAcl_ACL_TYPE */ -var ACL_TYPE_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4), 10): SONIC_ACL_TYPE_IPV4, - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6), 10): SONIC_ACL_TYPE_IPV6, - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2), 10): SONIC_ACL_TYPE_L2, -} - -/* E_OpenconfigAcl_FORWARDING_ACTION */ -var ACL_FORWARDING_ACTION_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_ACCEPT), 10): "FORWARD", - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_DROP), 10): "DROP", - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_REJECT), 10): "REDIRECT", -} - -/* E_OpenconfigPacketMatchTypes_IP_PROTOCOL */ -var IP_PROTOCOL_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_ICMP), 10): "1", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_IGMP), 10): "2", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_TCP), 10): "6", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_UDP), 10): "17", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_RSVP), 10): "46", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_GRE), 10): "47", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_AUTH), 10): "51", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_PIM), 10): "103", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_L2TP), 10): "115", -} - -var ETHERTYPE_MAP = map[ocbinds.E_OpenconfigPacketMatchTypes_ETHERTYPE]uint32{ - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_LLDP: 0x88CC, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_VLAN: 0x8100, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_ROCE: 0x8915, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_ARP: 0x0806, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_IPV4: 0x0800, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_IPV6: 0x86DD, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_MPLS: 0x8847, -} - -func getAclRoot(s *ygot.GoStruct) *ocbinds.OpenconfigAcl_Acl { - deviceObj := (*s).(*ocbinds.Device) - return deviceObj.Acl -} - -func getAclTypeOCEnumFromName(val string) (ocbinds.E_OpenconfigAcl_ACL_TYPE, error) { - switch val { - case "ACL_IPV4", "openconfig-acl:ACL_IPV4": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4, nil - case "ACL_IPV6", "openconfig-acl:ACL_IPV6": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6, nil - case "ACL_L2", "openconfig-acl:ACL_L2": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2, nil - default: - return ocbinds.OpenconfigAcl_ACL_TYPE_UNSET, - tlerr.NotSupported("ACL Type '%s' not supported", val) - } -} -func getAclKeyStrFromOCKey(aclname string, acltype ocbinds.E_OpenconfigAcl_ACL_TYPE) string { - aclN := strings.Replace(strings.Replace(aclname, " ", "_", -1), "-", "_", -1) - aclT := acltype.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(acltype)].Name - return aclN + "_" + aclT -} - -func getOCAclKeysFromStrDBKey(aclKey string) (string, ocbinds.E_OpenconfigAcl_ACL_TYPE) { - var aclOrigName string - var aclOrigType ocbinds.E_OpenconfigAcl_ACL_TYPE - - if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV4) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4 - } else if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV6) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6 - } else if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_L2) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2 - } - - return aclOrigName, aclOrigType -} - -func getTransportConfigTcpFlags(tcpFlags string) []ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS { - var flags []ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS - if len(tcpFlags) > 0 { - flagStr := strings.Split(tcpFlags, "/")[0] - flagNumber, _ := strconv.ParseUint(strings.Replace(flagStr, "0x", "", -1), 16, 32) - for i := 0; i < 8; i++ { - mask := 1 << uint(i) - if (int(flagNumber) & mask) > 0 { - switch int(flagNumber) & mask { - case 0x01: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_FIN) - case 0x02: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_SYN) - case 0x04: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_RST) - case 0x08: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_PSH) - case 0x10: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ACK) - case 0x20: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_URG) - case 0x40: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ECE) - case 0x80: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_CWR) - default: - } - } - } - } - return flags -} - -func getL2EtherType(etherType uint64) interface{} { - for k, v := range ETHERTYPE_MAP { - if uint32(etherType) == v { - return k - } - } - return uint16(etherType) -} - -//////////////////////////////////////////// -// Validate callpoints -//////////////////////////////////////////// -var validate_ipv4 ValidateCallpoint = func(inParams XfmrParams) (bool) { - if strings.Contains(inParams.key, "ACL_IPV4") { - return true - } - return false -} -var validate_ipv6 ValidateCallpoint = func(inParams XfmrParams) (bool) { - if strings.Contains(inParams.key, "ACL_IPV6") { - return true - } - return false -} - -//////////////////////////////////////////// -// Post Transformer -//////////////////////////////////////////// -var acl_post_xfmr PostXfmrFunc = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - log.Info("In Post transformer") - //TODO: check if a default ACL Rule exists, else create one and update the resultMap with default rule - // Return will be the updated result map - return (*inParams.dbDataMap)[inParams.curDb], nil -} - -//////////////////////////////////////////// -// Bi-directoonal overloaded methods -//////////////////////////////////////////// -var YangToDb_acl_forwarding_action_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["PACKET_ACTION"] = "" - return res_map, err - } - action, _ := inParams.param.(ocbinds.E_OpenconfigAcl_FORWARDING_ACTION) - log.Info("YangToDb_acl_forwarding_action_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " forwarding_action: ", action) - res_map["PACKET_ACTION"] = findInMap(ACL_FORWARDING_ACTION_MAP, strconv.FormatInt(int64(action), 10)) - return res_map, err -} -var DbToYang_acl_forwarding_action_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_forwarding_action_xfmr", data, inParams.ygRoot) - oc_action := findInMap(ACL_FORWARDING_ACTION_MAP, data[RULE_TABLE][inParams.key].Field["PACKET_ACTION"]) - n, err := strconv.ParseInt(oc_action, 10, 64) - result["forwarding-action"] = ocbinds.E_OpenconfigAcl_FORWARDING_ACTION(n).ΛMap()["E_OpenconfigAcl_FORWARDING_ACTION"][n].Name - return result, err -} - -var YangToDb_acl_type_field_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map[ACL_TYPE] = "" - return res_map, err - } - - acltype, _ := inParams.param.(ocbinds.E_OpenconfigAcl_ACL_TYPE) - log.Info("YangToDb_acl_type_field_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " acltype: ", acltype) - res_map[ACL_TYPE] = findInMap(ACL_TYPE_MAP, strconv.FormatInt(int64(acltype), 10)) - return res_map, err -} -var DbToYang_acl_type_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_type_field_xfmr", data, inParams.ygRoot) - oc_acltype := findInMap(ACL_TYPE_MAP, data[ACL_TABLE][inParams.key].Field[ACL_TYPE]) - n, err := strconv.ParseInt(oc_acltype, 10, 64) - result[ACL_TYPE] = ocbinds.E_OpenconfigAcl_ACL_TYPE(n).ΛMap()["E_OpenconfigAcl_ACL_TYPE"][n].Name - return result, err -} - -var YangToDb_acl_set_key_xfmr KeyXfmrYangToDb = func(inParams XfmrParams) (string, error) { - var aclkey string - var err error - var oc_aclType ocbinds.E_OpenconfigAcl_ACL_TYPE - log.Info("YangToDb_acl_set_key_xfmr: ", inParams.ygRoot, inParams.uri) - pathInfo := NewPathInfo(inParams.uri) - - if len(pathInfo.Vars) < 2 { - err = errors.New("Invalid xpath, key attributes not found") - return aclkey, err - } - - oc_aclType, err = getAclTypeOCEnumFromName(pathInfo.Var("type")) - if err != nil { - err = errors.New("OC Acl type name to OC Acl Enum failed") - return aclkey, err - } - - aclkey = getAclKeyStrFromOCKey(pathInfo.Var("name"), oc_aclType) - log.Info("YangToDb_acl_set_key_xfmr - acl_set_key : ", aclkey) - - return aclkey, err -} - -var DbToYang_acl_set_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { - rmap := make(map[string]interface{}) - var err error - var aclNameStr string - var aclTypeStr string - aclkey := inParams.key - log.Info("DbToYang_acl_set_key_xfmr: ", aclkey) - if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV4) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclTypeStr = "ACL_IPV4" - } else if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV6) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclTypeStr = "ACL_IPV6" - } else if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_L2) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclTypeStr = "ACL_L2" - } else { - err = errors.New("Invalid key for acl set.") - log.Info("Invalid Keys for acl acl set", aclkey) - } - rmap["name"] = aclNameStr - rmap["type"] = aclTypeStr - return rmap, err -} - -var DbToYang_acl_set_name_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - var err error - log.Info("DbToYang_acl_set_name_xfmr: ", inParams.key) - /*name attribute corresponds to key in redis table*/ - aclName, _ := getOCAclKeysFromStrDBKey(inParams.key) - res_map["name"] = aclName - log.Info("acl-set/config/name ", res_map) - return res_map, err -} - -var YangToDb_acl_entry_key_xfmr KeyXfmrYangToDb = func(inParams XfmrParams) (string, error) { - var entry_key string - var err error - var oc_aclType ocbinds.E_OpenconfigAcl_ACL_TYPE - log.Info("YangToDb_acl_entry_key_xfmr: ", inParams.ygRoot, inParams.uri) - pathInfo := NewPathInfo(inParams.uri) - - if len(pathInfo.Vars) < 3 { - err = errors.New("Invalid xpath, key attributes not found") - return entry_key, err - } - - oc_aclType, err = getAclTypeOCEnumFromName(pathInfo.Var("type")) - if err != nil { - err = errors.New("OC Acl type name to OC Acl Enum failed") - return entry_key, err - } - - aclkey := getAclKeyStrFromOCKey(pathInfo.Var("name"), oc_aclType) - var rulekey string - if strings.Contains(pathInfo.Template, "/acl-entry{sequence-id}") { - rulekey = "RULE_" + pathInfo.Var("sequence-id") - } - entry_key = aclkey + "|" + rulekey - - log.Info("YangToDb_acl_entry_key_xfmr - entry_key : ", entry_key) - - return entry_key, err -} - -var DbToYang_acl_entry_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { - rmap := make(map[string]interface{}) - var err error - entry_key := inParams.key - log.Info("DbToYang_acl_entry_key_xfmr: ", entry_key) - - key := strings.Split(entry_key, "|") - if len(key) < 2 { - err = errors.New("Invalid key for acl entries.") - log.Info("Invalid Keys for acl enmtries", entry_key) - return rmap, err - } - - dbAclRule := key[1] - seqId := strings.Replace(dbAclRule, "RULE_", "", 1) - rmap["sequence-id"], _ = strconv.ParseFloat(seqId, 64) - return rmap, err -} - -var DbToYang_acl_entry_sequenceid_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - var err error - log.Info("DbToYang_acl_entry_sequenceid_xfmr: ", inParams.key) - /*sequenec-id attribute corresponds to key in redis table*/ - res, err := DbToYang_acl_entry_key_xfmr(inParams) - log.Info("acl-entry/config/sequence-id ", res) - if err != nil { - return res_map, err - } - if seqId, ok := res["sequence-id"]; !ok { - log.Error("sequence-id not found in acl entry") - return res_map, err - } else { - res_map["sequence-id"] = seqId - } - return res_map, err -} - -var YangToDb_acl_l2_ethertype_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - - if inParams.param == nil { - res_map["ETHER_TYPE"] = "" - return res_map, err - } - ethertypeType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " ethertypeType: ", ethertypeType) - var b bytes.Buffer - switch ethertypeType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_E_OpenconfigPacketMatchTypes_ETHERTYPE{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_E_OpenconfigPacketMatchTypes_ETHERTYPE) - fmt.Fprintf(&b, "0x%0.4x", ETHERTYPE_MAP[v.E_OpenconfigPacketMatchTypes_ETHERTYPE]) - res_map["ETHER_TYPE"] = b.String() - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_Uint16) - fmt.Fprintf(&b, "0x%0.4x", v.Uint16) - res_map["ETHER_TYPE"] = b.String() - break - } - return res_map, err -} - -var DbToYang_acl_l2_ethertype_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_l2_ethertype_xfmr", data, inParams.ygRoot) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - etype, ok := ruleInst.Field["ETHER_TYPE"] - - if ok { - etypeVal, _ := strconv.ParseUint(strings.Replace(etype, "0x", "", -1), 16, 32) - result["protocol"] = getL2EtherType(etypeVal) - } else { - err = errors.New("ETHER_TYPE field not found in DB") - } - return result, nil -} - -var YangToDb_acl_ip_protocol_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - - if inParams.param == nil { - res_map["IP_PROTOCOL"] = "" - return res_map, err - } - protocolType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " protocolType: ", protocolType) - switch protocolType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_E_OpenconfigPacketMatchTypes_IP_PROTOCOL{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_E_OpenconfigPacketMatchTypes_IP_PROTOCOL) - res_map["IP_PROTOCOL"] = findInMap(IP_PROTOCOL_MAP, strconv.FormatInt(int64(v.E_OpenconfigPacketMatchTypes_IP_PROTOCOL), 10)) - v = nil - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_Uint8{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_Uint8) - res_map["IP_PROTOCOL"] = strconv.FormatInt(int64(v.Uint8), 10) - break - } - return res_map, err -} - -var DbToYang_acl_ip_protocol_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_ip_protocol_xfmr", data, inParams.ygRoot) - oc_protocol := findByValue(IP_PROTOCOL_MAP, data[RULE_TABLE][inParams.key].Field["IP_PROTOCOL"]) - n, err := strconv.ParseInt(oc_protocol, 10, 64) - result["protocol"] = ocbinds.E_OpenconfigPacketMatchTypes_IP_PROTOCOL(n).ΛMap()["E_OpenconfigPacketMatchTypes_IP_PROTOCOL"][n].Name - return result, err -} - -var YangToDb_acl_source_port_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["L4_SRC_PORT"] = "" - return res_map, err - } - sourceportType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " sourceportType: ", sourceportType) - switch sourceportType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort) - res_map["L4_SRC_PORT"] = v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort.ΛMap()["E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort"][int64(v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort)].Name - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_String{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_String) - res_map["L4_SRC_PORT_RANGE"] = strings.Replace(v.String, "..", "-", 1) - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_Uint16) - res_map["L4_SRC_PORT"] = strconv.FormatInt(int64(v.Uint16), 10) - break - } - return res_map, err -} - -var DbToYang_acl_source_port_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_source_port_xfmr: ", data, inParams.ygRoot) - result := make(map[string]interface{}) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - port, ok := ruleInst.Field["L4_SRC_PORT"] - if ok { - result["source-port"] = port - return result, nil - } - - portRange, ok := ruleInst.Field["L4_SRC_PORT_RANGE"] - if ok { - result["source-port"] = portRange - return result, nil - } else { - err = errors.New("PORT/PORT_RANGE field not found in DB") - } - return result, err -} - -var YangToDb_acl_destination_port_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["L4_DST_PORT_RANGE"] = "" - return res_map, err - } - destportType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " destportType: ", destportType) - switch destportType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort) - res_map["L4_DST_PORT"] = v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort.ΛMap()["E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort"][int64(v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort)].Name - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_String{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_String) - res_map["L4_DST_PORT_RANGE"] = strings.Replace(v.String, "..", "-", 1) - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_Uint16) - res_map["L4_DST_PORT"] = strconv.FormatInt(int64(v.Uint16), 10) - break - } - return res_map, err -} - -var DbToYang_acl_destination_port_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_destination_port_xfmr: ", data, inParams.ygRoot) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - port, ok := ruleInst.Field["L4_DST_PORT"] - if ok { - result["destination-port"] = port - return result, nil - } - - portRange, ok := ruleInst.Field["L4_DST_PORT_RANGE"] - if ok { - result["destination-port"] = portRange - return result, nil - } else { - err = errors.New("DST PORT/PORT_RANGE field not found in DB") - } - return result, err -} - -var YangToDb_acl_tcp_flags_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - log.Info("YangToDb_acl_tcp_flags_xfmr: ") - var tcpFlags uint32 = 0x00 - var b bytes.Buffer - if inParams.param == nil { - res_map["TCP_FLAGS"] = b.String() - return res_map, err - } - log.Info("YangToDb_acl_tcp_flags_xfmr: ", inParams.ygRoot, inParams.uri) - v := reflect.ValueOf(inParams.param) - - flags := v.Interface().([]ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS) - for _, flag := range flags { - fmt.Println("TCP Flag name: " + flag.ΛMap()["E_OpenconfigPacketMatchTypes_TCP_FLAGS"][int64(flag)].Name) - switch flag { - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_FIN: - tcpFlags |= 0x01 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_SYN: - tcpFlags |= 0x02 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_RST: - tcpFlags |= 0x04 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_PSH: - tcpFlags |= 0x08 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ACK: - tcpFlags |= 0x10 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_URG: - tcpFlags |= 0x20 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ECE: - tcpFlags |= 0x40 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_CWR: - tcpFlags |= 0x80 - break - } - } - fmt.Fprintf(&b, "0x%0.2x/0x%0.2x", tcpFlags, tcpFlags) - res_map["TCP_FLAGS"] = b.String() - return res_map, err -} - -var DbToYang_acl_tcp_flags_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_tcp_flags_xfmr: ", data, inParams.ygRoot) - result := make(map[string]interface{}) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - tcpFlag, ok := ruleInst.Field["TCP_FLAGS"] - if ok { - result["tcp-flags"] = getTransportConfigTcpFlags(tcpFlag) - return result, nil - } - return result, nil -} - -var YangToDb_acl_port_bindings_xfmr SubTreeXfmrYangToDb = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - var err error - res_map := make(map[string]map[string]db.Value) - aclTableMap := make(map[string]db.Value) - log.Info("YangToDb_acl_port_bindings_xfmr: ", inParams.ygRoot, inParams.uri) - - aclObj := getAclRoot(inParams.ygRoot) - if aclObj.Interfaces == nil { - return res_map, err - } - aclInterfacesMap := make(map[string][]string) - for intfId, _ := range aclObj.Interfaces.Interface { - intf := aclObj.Interfaces.Interface[intfId] - if intf != nil { - if intf.IngressAclSets != nil && len(intf.IngressAclSets.IngressAclSet) > 0 { - for inAclKey, _ := range intf.IngressAclSets.IngressAclSet { - aclName := getAclKeyStrFromOCKey(inAclKey.SetName, inAclKey.Type) - aclInterfacesMap[aclName] = append(aclInterfacesMap[aclName], *intf.Id) - _, ok := aclTableMap[aclName] - if !ok { - aclTableMap[aclName] = db.Value{Field: make(map[string]string)} - } - aclTableMap[aclName].Field["stage"] = "INGRESS" - } - } - if intf.EgressAclSets != nil && len(intf.EgressAclSets.EgressAclSet) > 0 { - for outAclKey, _ := range intf.EgressAclSets.EgressAclSet { - aclName := getAclKeyStrFromOCKey(outAclKey.SetName, outAclKey.Type) - aclInterfacesMap[aclName] = append(aclInterfacesMap[aclName], *intf.Id) - _, ok := aclTableMap[aclName] - if !ok { - aclTableMap[aclName] = db.Value{Field: make(map[string]string)} - } - aclTableMap[aclName].Field["stage"] = "EGRESS" - } - } - } - } - for k, _ := range aclInterfacesMap { - val := aclTableMap[k] - (&val).SetList("ports", aclInterfacesMap[k]) - } - res_map[ACL_TABLE] = aclTableMap - return res_map, err -} - -var DbToYang_acl_port_bindings_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) error { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_port_bindings_xfmr: ", data, inParams.ygRoot) - - aclTbl := data["ACL_TABLE"] - var ruleTbl map[string]map[string]db.Value - - // repopulate to use existing code - ruleTbl = make(map[string]map[string]db.Value) - for key, element := range data["ACL_RULE"] { - // split into aclKey and ruleKey - tokens := strings.Split(key, "|") - if ruleTbl[tokens[0]] == nil { - ruleTbl[tokens[0]] = make(map[string]db.Value) - } - ruleTbl[tokens[0]][tokens[1]] = db.Value{Field: make(map[string]string)} - ruleTbl[tokens[0]][tokens[1]] = element - } - - pathInfo := NewPathInfo(inParams.uri) - - acl := getAclRoot(inParams.ygRoot) - targetUriPath, _ := getYangPathFromUri(pathInfo.Path) - if isSubtreeRequest(pathInfo.Template, "/openconfig-acl:acl/interfaces/interface{}") { - for intfId := range acl.Interfaces.Interface { - intfData := acl.Interfaces.Interface[intfId] - ygot.BuildEmptyTree(intfData) - if isSubtreeRequest(targetUriPath, "/openconfig-acl:acl/interfaces/interface/ingress-acl-sets") { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "INGRESS") - } else if isSubtreeRequest(targetUriPath, "/openconfig-acl:acl/interfaces/interface/egress-acl-sets") { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "EGRESS") - } else { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "INGRESS") - if err != nil { - return err - } - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "EGRESS") - } - } - } else { - err = getAllBindingsInfo(aclTbl, ruleTbl, inParams.ygRoot) - } - - return err -} - -func convertInternalToOCAclRuleBinding(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, priority uint32, seqId int64, direction string, aclSet ygot.GoStruct, entrySet ygot.GoStruct) { - if seqId == -1 { - seqId = int64(MAX_PRIORITY - priority) - } - - var num uint64 - num = 0 - var ruleId uint32 = uint32(seqId) - - if direction == "INGRESS" { - var ingressEntrySet *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_AclEntries_AclEntry - var ok bool - if entrySet == nil { - ingressAclSet := aclSet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet) - if ingressEntrySet, ok = ingressAclSet.AclEntries.AclEntry[ruleId]; !ok { - ingressEntrySet, _ = ingressAclSet.AclEntries.NewAclEntry(ruleId) - } - } else { - ingressEntrySet = entrySet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_AclEntries_AclEntry) - } - if ingressEntrySet != nil { - ygot.BuildEmptyTree(ingressEntrySet) - ingressEntrySet.State.SequenceId = &ruleId - ingressEntrySet.State.MatchedPackets = &num - ingressEntrySet.State.MatchedOctets = &num - } - } else if direction == "EGRESS" { - var egressEntrySet *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_AclEntries_AclEntry - var ok bool - if entrySet == nil { - egressAclSet := aclSet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet) - if egressEntrySet, ok = egressAclSet.AclEntries.AclEntry[ruleId]; !ok { - egressEntrySet, _ = egressAclSet.AclEntries.NewAclEntry(ruleId) - } - } else { - egressEntrySet = entrySet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_AclEntries_AclEntry) - } - if egressEntrySet != nil { - ygot.BuildEmptyTree(egressEntrySet) - egressEntrySet.State.SequenceId = &ruleId - egressEntrySet.State.MatchedPackets = &num - egressEntrySet.State.MatchedOctets = &num - } - } -} - -func convertInternalToOCAclBinding(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, aclName string, intfId string, direction string, intfAclSet ygot.GoStruct) error { - var err error - if _, ok := aclTableMap[aclName]; !ok { - err = errors.New("Acl entry not found, convertInternalToOCAclBinding") - return err - } else { - aclEntry := aclTableMap[aclName] - if !contains(aclEntry.GetList("ports"), intfId) { - return tlerr.InvalidArgs("Acl %s not binded with %s", aclName, intfId) - } - } - - for ruleName := range ruleTableMap[aclName] { - if ruleName != "DEFAULT_RULE" { - seqId, _ := strconv.Atoi(strings.Replace(ruleName, "RULE_", "", 1)) - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, intfAclSet, nil) - } - } - - return err -} - -func getAllBindingsInfo(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, ygRoot *ygot.GoStruct) error { - var err error - acl := getAclRoot(ygRoot) - - var interfaces []string - for aclName := range aclTableMap { - aclData := aclTableMap[aclName] - if len(aclData.Get("ports@")) > 0 { - aclIntfs := aclData.GetList("ports") - for i, _ := range aclIntfs { - if !contains(interfaces, aclIntfs[i]) && aclIntfs[i] != "" { - interfaces = append(interfaces, aclIntfs[i]) - } - } - } - } - ygot.BuildEmptyTree(acl) - for _, intfId := range interfaces { - var intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface - intfData, ok := acl.Interfaces.Interface[intfId] - if !ok { - intfData, _ = acl.Interfaces.NewInterface(intfId) - } - ygot.BuildEmptyTree(intfData) - err = getAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfData, intfId, "INGRESS") - err = getAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfData, intfId, "EGRESS") - } - return err -} - -func getAclBindingInfoForInterfaceData(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface, intfId string, direction string) error { - var err error - if intfData != nil { - intfData.Config.Id = intfData.Id - intfData.State.Id = intfData.Id - } - if direction == "INGRESS" { - if intfData.IngressAclSets != nil && len(intfData.IngressAclSets.IngressAclSet) > 0 { - for ingressAclSetKey, _ := range intfData.IngressAclSets.IngressAclSet { - aclName := strings.Replace(strings.Replace(ingressAclSetKey.SetName, " ", "_", -1), "-", "_", -1) - aclType := ingressAclSetKey.Type.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(ingressAclSetKey.Type)].Name - aclKey := aclName + "_" + aclType - - ingressAclSet := intfData.IngressAclSets.IngressAclSet[ingressAclSetKey] - if ingressAclSet != nil && ingressAclSet.AclEntries != nil && len(ingressAclSet.AclEntries.AclEntry) > 0 { - for seqId, _ := range ingressAclSet.AclEntries.AclEntry { - rulekey := "RULE_" + strconv.Itoa(int(seqId)) - entrySet := ingressAclSet.AclEntries.AclEntry[seqId] - _, ok := ruleTableMap[aclKey+"|"+rulekey] - if !ok { - log.Info("Acl Rule not found ", aclKey, rulekey) - err = errors.New("Acl Rule not found ingress, getAclBindingInfoForInterfaceData") - return err - } - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, nil, entrySet) - } - } else { - ygot.BuildEmptyTree(ingressAclSet) - ingressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Config{SetName: &aclName, Type: ingressAclSetKey.Type} - ingressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_State{SetName: &aclName, Type: ingressAclSetKey.Type} - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclKey, intfId, direction, ingressAclSet) - } - } - } else { - err = findAndGetAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfId, direction, intfData) - } - } else if direction == "EGRESS" { - if intfData.EgressAclSets != nil && len(intfData.EgressAclSets.EgressAclSet) > 0 { - for egressAclSetKey, _ := range intfData.EgressAclSets.EgressAclSet { - aclName := strings.Replace(strings.Replace(egressAclSetKey.SetName, " ", "_", -1), "-", "_", -1) - aclType := egressAclSetKey.Type.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(egressAclSetKey.Type)].Name - aclKey := aclName + "_" + aclType - - egressAclSet := intfData.EgressAclSets.EgressAclSet[egressAclSetKey] - if egressAclSet != nil && egressAclSet.AclEntries != nil && len(egressAclSet.AclEntries.AclEntry) > 0 { - for seqId, _ := range egressAclSet.AclEntries.AclEntry { - rulekey := "RULE_" + strconv.Itoa(int(seqId)) - entrySet := egressAclSet.AclEntries.AclEntry[seqId] - _, ok := ruleTableMap[aclKey+"|"+rulekey] - if !ok { - log.Info("Acl Rule not found ", aclKey, rulekey) - err = errors.New("Acl Rule not found egress, getAclBindingInfoForInterfaceData") - return err - } - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, nil, entrySet) - } - } else { - ygot.BuildEmptyTree(egressAclSet) - egressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Config{SetName: &aclName, Type: egressAclSetKey.Type} - egressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_State{SetName: &aclName, Type: egressAclSetKey.Type} - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclKey, intfId, direction, egressAclSet) - } - } - } else { - err = findAndGetAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfId, direction, intfData) - } - } else { - log.Error("Unknown direction") - } - return err -} - -func findAndGetAclBindingInfoForInterfaceData(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, intfId string, direction string, intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface) error { - var err error - for aclName, _ := range aclTableMap { - aclData := aclTableMap[aclName] - aclIntfs := aclData.GetList("ports") - aclType := aclData.Get(ACL_TYPE) - var aclOrigName string - var aclOrigType ocbinds.E_OpenconfigAcl_ACL_TYPE - if SONIC_ACL_TYPE_IPV4 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4 - } else if SONIC_ACL_TYPE_IPV6 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6 - } else if SONIC_ACL_TYPE_L2 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2 - } - - if contains(aclIntfs, intfId) && direction == aclData.Get("stage") { - if direction == "INGRESS" { - if intfData.IngressAclSets != nil { - aclSetKey := ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Key{SetName: aclOrigName, Type: aclOrigType} - ingressAclSet, ok := intfData.IngressAclSets.IngressAclSet[aclSetKey] - if !ok { - ingressAclSet, _ = intfData.IngressAclSets.NewIngressAclSet(aclOrigName, aclOrigType) - ygot.BuildEmptyTree(ingressAclSet) - ingressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Config{SetName: &aclOrigName, Type: aclOrigType} - ingressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_State{SetName: &aclOrigName, Type: aclOrigType} - } - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclName, intfId, direction, ingressAclSet) - if err != nil { - return err - } - } - } else if direction == "EGRESS" { - if intfData.EgressAclSets != nil { - aclSetKey := ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Key{SetName: aclOrigName, Type: aclOrigType} - egressAclSet, ok := intfData.EgressAclSets.EgressAclSet[aclSetKey] - if !ok { - egressAclSet, _ = intfData.EgressAclSets.NewEgressAclSet(aclOrigName, aclOrigType) - ygot.BuildEmptyTree(egressAclSet) - egressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Config{SetName: &aclOrigName, Type: aclOrigType} - egressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_State{SetName: &aclOrigName, Type: aclOrigType} - } - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclName, intfId, direction, egressAclSet) - if err != nil { - return err - } - } - } - } - } - return err -} diff --git a/translib/transformer/xfmr_dbtbl_cbk_interface.go b/translib/transformer/xfmr_dbtbl_cbk_interface.go new file mode 100644 index 000000000000..8b2ffdea4899 --- /dev/null +++ b/translib/transformer/xfmr_dbtbl_cbk_interface.go @@ -0,0 +1,53 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" +) + +type XfmrDbTblCbkParams struct { + d *db.DB //Config DB handler + oper int + delDepRefKey string + tblName string + dbKey string + delDepEntry map[string]string + dbDataMap map[db.DBNum]map[string]map[string]db.Value + delDepDataMap map[int]*RedisDbMap // Call back methods can add the data +} + +func formXfmrDbTblCbkParams (d *db.DB, oper int, delDepRefKey string, tblName string, dbKey string, delDepEntry map[string]string, dbDataMap RedisDbMap) XfmrDbTblCbkParams { + + var inParams XfmrDbTblCbkParams + + inParams.d = d + inParams.oper = oper + inParams.delDepRefKey = delDepRefKey + inParams.tblName = tblName + inParams.dbKey = dbKey + inParams.delDepEntry = delDepEntry + inParams.dbDataMap = dbDataMap + inParams.delDepDataMap = make(map[int]*RedisDbMap) + + return inParams +} + +type XfmrDbTblCbkMethod func (inParams XfmrDbTblCbkParams) error + diff --git a/translib/transformer/xfmr_interface.go b/translib/transformer/xfmr_interface.go index 57bc650c8c04..f705d0dcba93 100644 --- a/translib/transformer/xfmr_interface.go +++ b/translib/transformer/xfmr_interface.go @@ -21,115 +21,175 @@ package transformer import ( "github.com/openconfig/ygot/ygot" "github.com/Azure/sonic-mgmt-common/translib/db" - log "github.com/golang/glog" + "sync" ) +type RedisDbMap = map[db.DBNum]map[string]map[string]db.Value + +// XfmrParams represents input parameters for table-transformer, key-transformer, field-transformer & subtree-transformer type XfmrParams struct { d *db.DB dbs [db.MaxDB]*db.DB curDb db.DBNum ygRoot *ygot.GoStruct uri string + requestUri string //original uri using which a curl/NBI request is made oper int + table string key string dbDataMap *map[db.DBNum]map[string]map[string]db.Value + subOpDataMap map[int]*RedisDbMap // used to add an in-flight data with a sub-op param interface{} + txCache *sync.Map + skipOrdTblChk *bool + isVirtualTbl *bool + pCascadeDelTbl *[] string //used to populate list of tables needed cascade delete by subtree overloaded methods + yangDefValMap map[string]map[string]db.Value +} + +// SubscProcType represents subcription process type identifying the type of subscription request made from translib. +type SubscProcType int +const ( + TRANSLATE_SUBSCRIBE SubscProcType = iota + PROCESS_SUBSCRIBE +) + +/*susbcription sampling interval and subscription preference type*/ +type notificationOpts struct { + mInterval int + pType NotificationType +} + +// XfmrSubscInParams represents input to subscribe subtree callbacks - request uri, DBs info access-pointers, DB info for request uri and subscription process type from translib. +type XfmrSubscInParams struct { + uri string + dbs [db.MaxDB]*db.DB + dbDataMap RedisDbMap + subscProc SubscProcType +} + +// XfmrSubscOutParams represents output from subscribe subtree callback - DB data for request uri, Need cache, OnChange, subscription preference and interval. +type XfmrSubscOutParams struct { + dbDataMap RedisDbMap + needCache bool + onChange bool + nOpts *notificationOpts //these can be set regardless of error + isVirtualTbl bool //used for RFC parent table check, set to true when no Redis Mapping +} + +// XfmrDbParams represents input paraDeters for value-transformer +type XfmrDbParams struct { + oper int + dbNum db.DBNum + tableName string + key string + fieldName string + value string } -/** - * KeyXfmrYangToDb type is defined to use for conversion of Yang key to DB Key - * Transformer function definition. - * Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath - * Return: Database keys to access db entry, error - **/ + +// KeyXfmrYangToDb type is defined to use for conversion of Yang key to DB Key, +// Transformer function definition. +// Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath +// Return: Database keys to access db entry, error type KeyXfmrYangToDb func (inParams XfmrParams) (string, error) -/** - * KeyXfmrDbToYang type is defined to use for conversion of DB key to Yang key - * Transformer function definition. - * Param: XfmrParams structure having Database info, operation, Database keys to access db entry - * Return: multi dimensional map to hold the yang key attributes of complete xpath, error - **/ + +// KeyXfmrDbToYang type is defined to use for conversion of DB key to Yang key, +// Transformer function definition. +// Param: XfmrParams structure having Database info, operation, Database keys to access db entry +// Return: multi dimensional map to hold the yang key attributes of complete xpath, error */ type KeyXfmrDbToYang func (inParams XfmrParams) (map[string]interface{}, error) -/** - * FieldXfmrYangToDb type is defined to use for conversion of yang Field to DB field - * Transformer function definition. - * Param: Database info, YgotRoot, operation, Xpath - * Return: multi dimensional map to hold the DB data, error - **/ +// FieldXfmrYangToDb type is defined to use for conversion of yang Field to DB field +// Transformer function definition. +// Param: Database info, YgotRoot, operation, Xpath +// Return: multi dimensional map to hold the DB data, error type FieldXfmrYangToDb func (inParams XfmrParams) (map[string]string, error) -/** - * FieldXfmrDbtoYang type is defined to use for conversion of DB field to Yang field - * Transformer function definition. - * Param: XfmrParams structure having Database info, operation, DB data in multidimensional map, output param YgotRoot - * Return: error - **/ + +// FieldXfmrDbtoYang type is defined to use for conversion of DB field to Yang field +// Transformer function definition. +// Param: XfmrParams structure having Database info, operation, DB data in multidimensional map, output param YgotRoot +// Return: error type FieldXfmrDbtoYang func (inParams XfmrParams) (map[string]interface{}, error) -/** - * SubTreeXfmrYangToDb type is defined to use for handling the yang subtree to DB - * Transformer function definition. - * Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath - * Return: multi dimensional map to hold the DB data, error - **/ +// SubTreeXfmrYangToDb type is defined to use for handling the yang subtree to DB +// Transformer function definition. +// Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath +// Return: multi dimensional map to hold the DB data, error type SubTreeXfmrYangToDb func (inParams XfmrParams) (map[string]map[string]db.Value, error) -/** - * SubTreeXfmrDbToYang type is defined to use for handling the DB to Yang subtree - * Transformer function definition. - * Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri - * Return : error - **/ + +// SubTreeXfmrDbToYang type is defined to use for handling the DB to Yang subtree +// Transformer function definition. +// Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri +// Return : error type SubTreeXfmrDbToYang func (inParams XfmrParams) (error) -/** - * ValidateCallpoint is used to validate a YANG node during data translation back to YANG as a response to GET - * Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri - * Return : bool - **/ + +// SubTreeXfmrSubscribe type is defined to use for handling subscribe(translateSubscribe & processSubscribe) subtree +// Transformer function definition. +// Param : XfmrSubscInParams structure having uri, database pointers, subcribe process(translate/processSusbscribe), DB data in multidimensional map +// Return : XfmrSubscOutParams structure (db data in multiD map, needCache, pType, onChange, minInterval), error +type SubTreeXfmrSubscribe func (inParams XfmrSubscInParams) (XfmrSubscOutParams, error) + +// ValidateCallpoint is used to validate a YANG node during data translation back to YANG as a response to GET +// Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri +// Return : bool type ValidateCallpoint func (inParams XfmrParams) (bool) -/** - * PostXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE - * Transformer function definition. - * Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri - * Return: multi dimensional map to hold the DB data, error - **/ -type PostXfmrFunc func (inParams XfmrParams) (map[string]map[string]db.Value, error) +// RpcCallpoint is used to invoke a callback for action +// Param : []byte input payload, dbi indices +// Return : []byte output payload, error +type RpcCallpoint func (body []byte, dbs [db.MaxDB]*db.DB) ([]byte, error) +// PostXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE +// Transformer function definition. +// Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri +// Return: Multi dimensional map to hold the DB data Map (tblName, key and Fields), error +type PostXfmrFunc func (inParams XfmrParams) (map[string]map[string]db.Value, error) -/** - * TableXfmrFunc type is defined to use for table transformer function for dynamic derviation of redis table. - * Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri - * Return: List of table names, error - **/ +// TableXfmrFunc type is defined to use for table transformer function for dynamic derviation of redis table. +// Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri +// Return: List of table names, error type TableXfmrFunc func (inParams XfmrParams) ([]string, error) +// ValueXfmrFunc type is defined to use for conversion of DB field value from one forma to another +// Transformer function definition. +// Param: XfmrDbParams structure having Database info, operation, db-number, table, key, field, value +// Return: value string, error +type ValueXfmrFunc func (inParams XfmrDbParams) (string, error) -/** - * Xfmr validation interface for validating the callback registration of app modules - * transformer methods. - **/ + // PreXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE, UPDATE, REPLACE, DELETE & GET + // Transformer function definition. + // Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri + // Return: error +type PreXfmrFunc func (inParams XfmrParams) (error) + +// XfmrInterface is a validation interface for validating the callback registration of app modules +// transformer methods. type XfmrInterface interface { xfmrInterfaceValiidate() } func (KeyXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for KeyXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for KeyXfmrYangToDb") } func (KeyXfmrDbToYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for KeyXfmrDbToYang") + xfmrLogInfo("xfmrInterfaceValiidate for KeyXfmrDbToYang") } func (FieldXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for FieldXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for FieldXfmrYangToDb") } func (FieldXfmrDbtoYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for FieldXfmrDbtoYang") + xfmrLogInfo("xfmrInterfaceValiidate for FieldXfmrDbtoYang") } func (SubTreeXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for SubTreeXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrYangToDb") } func (SubTreeXfmrDbToYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for SubTreeXfmrDbToYang") + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrDbToYang") +} +func (SubTreeXfmrSubscribe) xfmrInterfaceValiidate () { + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrSubscribe") } func (TableXfmrFunc) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for TableXfmrFunc") + xfmrLogInfo("xfmrInterfaceValiidate for TableXfmrFunc") } diff --git a/translib/transformer/xfmr_path_utils.go b/translib/transformer/xfmr_path_utils.go index 8a052cd24c63..6fbd8830e850 100644 --- a/translib/transformer/xfmr_path_utils.go +++ b/translib/transformer/xfmr_path_utils.go @@ -20,6 +20,12 @@ type PathInfo struct { Vars map[string]string } +// HasVar checks if the PathInfo contains given variable. +func (p *PathInfo) HasVar(name string) bool { + _, exists := p.Vars[name] + return exists +} + // Var returns the string value for a path variable. Returns // empty string if no such variable exists. func (p *PathInfo) Var(name string) string { @@ -45,8 +51,17 @@ func NewPathInfo(path string) *PathInfo { name := readUntil(r, '=') value := readUntil(r, ']') + // Handle duplicate parameter names by suffixing "#N" to it. + // N is the number of occurance of that parameter name. + if info.HasVar(name) { + namePrefix := name + for k := 2; info.HasVar(name); k++ { + name = fmt.Sprintf("%s#%d", namePrefix, k) + } + } + if len(name) != 0 { - fmt.Fprintf(&template, "{%s}", name) + fmt.Fprintf(&template, "{}") info.Vars[name] = value } } @@ -57,17 +72,50 @@ func NewPathInfo(path string) *PathInfo { } func readUntil(r *strings.Reader, delim byte) string { - var buff strings.Builder - for { - c, err := r.ReadByte() - if err == nil && c != delim { - buff.WriteByte(c) - } else { - break + var buff strings.Builder + var escaped bool + + for { + c, err := r.ReadByte() + if err != nil || (c == delim && !escaped) { + break + } else if c == '\\' && !escaped { + escaped = true + } else { + escaped = false + buff.WriteByte(c) + } + } + + return buff.String() +} + +// SplitPath splits the ygot path into parts. +func SplitPath(path string) []string { + var parts []string + var start int + var inEscape, inKey bool + + path = strings.TrimPrefix(path, "/") + + for i, c := range path { + switch { + case inEscape: + inEscape = false + case c == '\'': + inEscape = true + case c == '[': + inKey = true + case c == ']': + inKey = false + case c == '/' && !inEscape && !inKey: + parts = append(parts, path[start:i]) + start = i + 1 } } - return buff.String() + parts = append(parts, path[start:]) + return parts } func RemoveXPATHPredicates(s string) (string, error) { diff --git a/translib/transformer/xlate.go b/translib/transformer/xlate.go index 20d80a1621e8..3317eaaebdcc 100644 --- a/translib/transformer/xlate.go +++ b/translib/transformer/xlate.go @@ -28,6 +28,7 @@ import ( "strings" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" ) const ( @@ -36,15 +37,10 @@ const ( REPLACE UPDATE DELETE + SUBSCRIBE + MAXOPER ) -type KeySpec struct { - dbNum db.DBNum - Ts db.TableSpec - Key db.Key - Child []KeySpec -} - var XlateFuncs = make(map[string]reflect.Value) var ( @@ -63,18 +59,24 @@ func XlateFuncBind(name string, fn interface{}) (err error) { v.Type().NumIn() XlateFuncs[name] = v } else { - log.Info("Duplicate entry found in the XlateFunc map " + name) + xfmrLogInfo("Duplicate entry found in the XlateFunc map " + name) } return } - +func IsXlateFuncBinded(name string) bool { + if _, ok := XlateFuncs[name]; !ok { + return false + } else { + return true + } +} func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, err error) { if _, ok := XlateFuncs[name]; !ok { - err = errors.New(name + " Xfmr function does not exist.") - return nil, err + log.Warning(name + " Xfmr function does not exist.") + return nil, nil } if len(params) != XlateFuncs[name].Type().NumIn() { - err = ErrParamsNotAdapted + log.Warning("Error parameters not adapted") return nil, nil } in := make([]reflect.Value, len(params)) @@ -85,53 +87,103 @@ func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, return result, nil } -func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key) error { +func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { + var dataMap = make(RedisDbMap) + + for i := db.ApplDB; i < db.MaxDB; i++ { + dataMap[i] = make(map[string]map[string]db.Value) + } + + err := traverseDbHelper(dbs, spec, &dataMap, parentKey, dbTblKeyGetCache) + if err != nil { + log.Warning("Couldn't get data from traverseDbHelper") + return err + } + /* db data processing */ + curMap := make(map[int]map[db.DBNum]map[string]map[string]db.Value) + curMap[GET] = dataMap + err = dbDataXfmrHandler(curMap) + if err != nil { + log.Warning("No conversion in dbdata-xfmr") + return err + } + + for oper, dbData := range curMap { + if oper == GET { + for dbNum, tblData := range dbData { + mapCopy((*result)[dbNum], tblData) + } + } + } + return nil +} + +func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { var err error - var dbOpts db.Options + var dbOpts db.Options = getDBOptions(spec.DbNum) - dbOpts = getDBOptions(spec.dbNum) separator := dbOpts.KeySeparator - log.Infof("key separator for table %v in Db %v is %v", spec.Ts.Name, spec.dbNum, separator) if spec.Key.Len() > 0 { // get an entry with a specific key - data, err := dbs[spec.dbNum].GetEntry(&spec.Ts, spec.Key) - if err != nil { - return err - } + if spec.Ts.Name != XFMR_NONE_STRING { // Do not traverse for NONE table + data, err := dbs[spec.DbNum].GetEntry(&spec.Ts, spec.Key) + queriedDbInfo := make(map[string]map[string]bool) + queriedDbTblInfo := make(map[string]bool) + queriedDbTblInfo[strings.Join(spec.Key.Comp, separator)] = true + queriedDbInfo[spec.Ts.Name] = queriedDbTblInfo + if dbTblKeyGetCache == nil { + dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) + } + dbTblKeyGetCache[spec.DbNum] = queriedDbInfo + if err != nil { + log.Warningf("Couldn't get data for tbl(%v), key(%v) in traverseDbHelper", spec.Ts.Name, spec.Key) + return err + } - if (*result)[spec.dbNum][spec.Ts.Name] == nil { - (*result)[spec.dbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} - } else { - (*result)[spec.dbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data + if (*result)[spec.DbNum][spec.Ts.Name] == nil { + (*result)[spec.DbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} + } else { + (*result)[spec.DbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data + } } - if len(spec.Child) > 0 { for _, ch := range spec.Child { - err = TraverseDb(dbs, ch, result, &spec.Key) + err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) } } } else { // TODO - GetEntry support with regex patten, 'abc*' for optimization - keys, err := dbs[spec.dbNum].GetKeys(&spec.Ts) - if err != nil { - return err - } - for i, _ := range keys { - if parentKey != nil { - // TODO - multi-depth with a custom delimiter - if strings.Index(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) == -1 { - continue + if spec.Ts.Name != XFMR_NONE_STRING { //Do not traverse for NONE table + keys, err := dbs[spec.DbNum].GetKeys(&spec.Ts) + if err != nil { + log.Warningf("Couldn't get keys for tbl(%v) in traverseDbHelper", spec.Ts.Name) + return err + } + xfmrLogInfoAll("keys for table %v in Db %v are %v", spec.Ts.Name, spec.DbNum, keys) + for i := range keys { + if parentKey != nil && !spec.IgnoreParentKey { + // TODO - multi-depth with a custom delimiter + if !strings.Contains(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) { + continue + } } + spec.Key = keys[i] + err = traverseDbHelper(dbs, spec, result, parentKey, dbTblKeyGetCache) + if err != nil { + log.Warningf("Traversal didn't fetch for : %v", err) + } } - spec.Key = keys[i] - err = TraverseDb(dbs, spec, result, parentKey) - } + } else if len(spec.Child) > 0 { + for _, ch := range spec.Child { + err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) + } + } } return err } -func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]KeySpec, error) { +func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}) (*[]KeySpec, error) { var err error var retdbFormat = make([]KeySpec, 0) @@ -140,17 +192,27 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke if isSonicYang(uri) { /* Extract the xpath and key from input xpath */ xpath, keyStr, tableName := sonicXpathKeyExtract(uri) + if tblSpecInfo, ok := xDbSpecMap[tableName]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + keyStr, err = dbKeyValueXfmrHandler(CREATE, tblSpecInfo.dbIndex, tableName, keyStr) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", tableName, keyStr) + return &retdbFormat, err + } + } + retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr) } else { /* Extract the xpath and key from input xpath */ - xpath, keyStr, _ := xpathKeyExtract(nil, ygRoot, 0, uri) - retdbFormat = FillKeySpecs(xpath, keyStr, &retdbFormat) + retData, _ := xpathKeyExtract(nil, ygRoot, GET, uri, requestUri, nil, nil, txCache, nil) + retdbFormat = FillKeySpecs(retData.xpath, retData.dbKey, &retdbFormat) } return &retdbFormat, err } func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]KeySpec){ + var err error if xYangSpecMap == nil { return *retdbFormat } @@ -160,11 +222,26 @@ func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]K if xpathInfo.tableName != nil { dbFormat := KeySpec{} dbFormat.Ts.Name = *xpathInfo.tableName - dbFormat.dbNum = xpathInfo.dbIndex + dbFormat.DbNum = xpathInfo.dbIndex + if len(xYangSpecMap[yangXpath].xfmrKey) > 0 || xYangSpecMap[yangXpath].keyName != nil { + dbFormat.IgnoreParentKey = true + } else { + dbFormat.IgnoreParentKey = false + } if keyStr != "" { + if tblSpecInfo, ok := xDbSpecMap[dbFormat.Ts.Name]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + keyStr, err = dbKeyValueXfmrHandler(CREATE, dbFormat.DbNum, dbFormat.Ts.Name, keyStr) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", dbFormat.Ts.Name, keyStr) + } + } dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } for _, child := range xpathInfo.childTable { + if child == dbFormat.Ts.Name { + continue + } if xDbSpecMap != nil { if _, ok := xDbSpecMap[child]; ok { chlen := len(xDbSpecMap[child].yangXpath) @@ -208,7 +285,7 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe if _, ok := xDbSpecMap[tableName]; ok { cdb = xDbSpecMap[tableName].dbIndex } - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb if keyStr != "" { dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } @@ -220,13 +297,13 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe if _, ok := xDbSpecMap[container]; ok { dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { - for dir, _ := range dbInfo.dbEntry.Dir { + for dir := range dbInfo.dbEntry.Dir { _, ok := xDbSpecMap[dir] if ok && xDbSpecMap[dir].dbEntry.Node.Statement().Keyword == "container" { cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb retdbFormat = append(retdbFormat, dbFormat) } } @@ -237,12 +314,14 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe return retdbFormat } -func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interface{}) (map[string]map[string]db.Value, error) { +func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interface{}, jsonPayload []byte, txCache interface{}, skipOrdTbl *bool) (map[int]RedisDbMap, map[string]map[string]db.Value, map[string]map[string]db.Value, error) { var err error + requestUri := path + jsonData := make(map[string]interface{}) device := (*yg).(*ocbinds.Device) - jsonStr, err := ygot.EmitJSON(device, &ygot.EmitJSONConfig{ + jsonStr, _ := ygot.EmitJSON(device, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, Indent: " ", SkipValidation: true, @@ -251,85 +330,106 @@ func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interfa }, }) - jsonData := make(map[string]interface{}) err = json.Unmarshal([]byte(jsonStr), &jsonData) if err != nil { - log.Errorf("Error: failed to unmarshal json.") - return nil, err + errStr := "Error: failed to unmarshal json." + err = tlerr.InternalError{Format: errStr} + return nil, nil, nil, err } // Map contains table.key.fields - var result = make(map[string]map[string]db.Value) + var result = make(map[int]RedisDbMap) + var yangDefValMap = make(map[string]map[string]db.Value) + var yangAuxValMap = make(map[string]map[string]db.Value) switch opcode { case CREATE: - log.Info("CREATE case") - err = dbMapCreate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("CREATE case") + err = dbMapCreate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for create request.") + log.Warning("Data translation from yang to db failed for create request.") } case UPDATE: - log.Info("UPDATE case") - err = dbMapUpdate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("UPDATE case") + err = dbMapUpdate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for update request.") + log.Warning("Data translation from yang to db failed for update request.") } case REPLACE: - log.Info("REPLACE case") - err = dbMapUpdate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("REPLACE case") + err = dbMapUpdate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for replace request.") + log.Warning("Data translation from yang to db failed for replace request.") } case DELETE: - log.Info("DELETE case") - err = dbMapDelete(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("DELETE case") + err = dbMapDelete(d, yg, opcode, path, requestUri, jsonData, result, txCache, skipOrdTbl) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for delete request.") + log.Warning("Data translation from yang to db failed for delete request.") } } - return result, err + return result, yangDefValMap, yangAuxValMap, err } -func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB) ([]byte, error) { +func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}) ([]byte, bool, error) { var err error var payload []byte - log.Info("received xpath =", uri) - - keySpec, err := XlateUriToKeySpec(uri, ygRoot, nil) - var dbresult = make(map[db.DBNum]map[string]map[string]db.Value) + var inParamsForGet xlateFromDbParams + xfmrLogInfo("received xpath = " + uri) + requestUri := uri + keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache) + var dbresult = make(RedisDbMap) for i := db.ApplDB; i < db.MaxDB; i++ { dbresult[i] = make(map[string]map[string]db.Value) } + inParamsForGet.dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) + for _, spec := range *keySpec { - err := TraverseDb(dbs, spec, &dbresult, nil) + err := TraverseDb(dbs, spec, &dbresult, nil, inParamsForGet.dbTblKeyGetCache) if err != nil { - log.Error("TraverseDb() failure") - return payload, err + log.Warning("TraverseDb() didn't fetch data.") } } - payload, err = XlateFromDb(uri, ygRoot, dbs, dbresult) + isEmptyPayload := false + payload, isEmptyPayload, err = XlateFromDb(uri, ygRoot, dbs, dbresult, txCache, inParamsForGet) if err != nil { - log.Error("XlateFromDb() failure.") - return payload, err + return payload, true, err } - return payload, err + return payload, isEmptyPayload, err } -func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data map[db.DBNum]map[string]map[string]db.Value) ([]byte, error) { +func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data RedisDbMap, txCache interface{}, inParamsForGet xlateFromDbParams) ([]byte, bool, error) { var err error var result []byte - var dbData = make(map[db.DBNum]map[string]map[string]db.Value) + var dbData = make(RedisDbMap) var cdb db.DBNum = db.ConfigDB + var xpath string dbData = data + requestUri := uri + /* Check if the parent table exists for RFC compliance */ + var exists bool + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(nil, dbs, ygRoot, GET, uri, dbData, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + log.Warningf("Cannot perform GET Operation on uri %v due to - %v", uri, err) + return []byte(""), true, err + } + if !exists { + err = tlerr.NotFoundError{Format:"Resource Not found"} + return []byte(""), true, err + } + if isSonicYang(uri) { - xpath, keyStr, tableName := sonicXpathKeyExtract(uri) + lxpath, keyStr, tableName := sonicXpathKeyExtract(uri) + xpath = lxpath if (tableName != "") { dbInfo, ok := xDbSpecMap[tableName] if !ok { @@ -340,51 +440,104 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m tokens:= strings.Split(xpath, "/") // Format /module:container/tableName/listname[key]/fieldName if tokens[SONIC_TABLE_INDEX] == tableName { - fieldName := tokens[len(tokens)-1] - dbSpecField := tableName + "/" + fieldName - _, ok := xDbSpecMap[dbSpecField] - if ok && xDbSpecMap[dbSpecField].fieldType == "leaf" { - dbData[cdb] = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + fieldName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + dbSpecField := tableName + "/" + fieldName + dbSpecFieldInfo, ok := xDbSpecMap[dbSpecField] + if ok && fieldName != "" { + yangNodeType := yangTypeGet(xDbSpecMap[dbSpecField].dbEntry) + if yangNodeType == YANG_LEAF_LIST { + fieldName = fieldName + "@" + } + if ((yangNodeType == YANG_LEAF_LIST) || (yangNodeType == YANG_LEAF)) { + dbData[cdb], err = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + // return resource not found when the leaf/leaf-list instance(not entire leaf-list GET) not found + if ((err != nil) && ((yangNodeType == YANG_LEAF) || ((yangNodeType == YANG_LEAF_LIST) && (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/"))))) { + return []byte(""), true, err + } + if ((yangNodeType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + leafListInstVal, valErr := extractLeafListInstFromUri(uri) + if valErr != nil { + return []byte(""), true, valErr + } + if dbSpecFieldInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(CREATE, cdb, tableName, keyStr, fieldName, leafListInstVal) + retVal, err := valueXfmrHandler(inParams, *dbSpecFieldInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", dbSpecField, leafListInstVal, err) + return []byte(""), true, err + } + log.Info("valueXfmrHandler() retuned ", retVal) + leafListInstVal = retVal + } + if leafListInstExists(dbData[cdb][tableName][keyStr].Field[fieldName], leafListInstVal) { + /* Since translib already fills in ygRoot with queried leaf-list instance, do not + fill in resFldValMap or else Unmarshall of payload(resFldValMap) into ygotTgt in + app layer will create duplicate instances in result. + */ + log.Info("Queried leaf-list instance exists.") + return []byte("{}"), false, nil + } else { + xfmrLogInfoAll("Queried leaf-list instance does not exist - %v", uri) + return []byte(""), true, tlerr.NotFoundError{Format:"Resource not found"} + } + } + } + } } } } } else { - xpath, _ := XfmrRemoveXPATHPredicates(uri) + lxpath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpath = lxpath if _, ok := xYangSpecMap[xpath]; ok { cdb = xYangSpecMap[xpath].dbIndex } } - payload, err := dbDataToYangJsonCreate(uri, ygRoot, dbs, &dbData, cdb) - log.Info("Payload generated:", payload) + dbTblKeyGetCache := inParamsForGet.dbTblKeyGetCache + inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false) + inParamsForGet.xfmrDbTblKeyCache = make(map[string]tblKeyCache) + inParamsForGet.dbTblKeyGetCache = dbTblKeyGetCache + payload, isEmptyPayload, err := dbDataToYangJsonCreate(inParamsForGet) + xfmrLogInfoAll("Payload generated : " + payload) if err != nil { - log.Errorf("Error: failed to create json response from DB data.") - return nil, err + log.Warning("Couldn't create json response from DB data.") + return nil, isEmptyPayload, err } + xfmrLogInfo("Created json response from DB data.") result = []byte(payload) - return result, err + return result, isEmptyPayload, err } -func extractFieldFromDb(tableName string, keyStr string, fieldName string, data map[string]map[string]db.Value) (map[string]map[string]db.Value) { +func extractFieldFromDb(tableName string, keyStr string, fieldName string, data map[string]map[string]db.Value) (map[string]map[string]db.Value, error) { var dbVal db.Value var dbData = make(map[string]map[string]db.Value) + var err error if tableName != "" && keyStr != "" && fieldName != "" { if data[tableName][keyStr].Field != nil { - dbData[tableName] = make(map[string]db.Value) - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = data[tableName][keyStr].Field[fieldName] - dbData[tableName][keyStr] = dbVal + fldVal, fldValExists := data[tableName][keyStr].Field[fieldName] + if fldValExists { + dbData[tableName] = make(map[string]db.Value) + dbVal.Field = make(map[string]string) + dbVal.Field[fieldName] = fldVal + dbData[tableName][keyStr] = dbVal + } else { + log.Warningf("Field %v doesn't exist in table - %v, instance - %v", fieldName, tableName, keyStr) + err = tlerr.NotFoundError{Format: "Resource not found"} + } } } - return dbData + return dbData, err } func GetModuleNmFromPath(uri string) (string, error) { - log.Infof("received uri %s to extract module name from ", uri) + xfmrLogInfo("received uri %s to extract module name from ", uri) moduleNm, err := uriModuleNameGet(uri) return moduleNm, err } @@ -395,13 +548,308 @@ func GetOrdDBTblList(ygModuleNm string) ([]string, error) { if dbTblList, ok := xDbSpecOrdTblMap[ygModuleNm]; ok { result = dbTblList if len(dbTblList) == 0 { - log.Error("Ordered DB Table list is empty for module name = ", ygModuleNm) + log.Warning("Ordered DB Table list is empty for module name = ", ygModuleNm) err = fmt.Errorf("Ordered DB Table list is empty for module name %v", ygModuleNm) } } else { - log.Error("No entry found in the map of module names to ordered list of DB Tables for module = ", ygModuleNm) + log.Warning("No entry found in the map of module names to ordered list of DB Tables for module = ", ygModuleNm) err = fmt.Errorf("No entry found in the map of module names to ordered list of DB Tables for module = %v", ygModuleNm) } return result, err } + +func GetOrdTblList(xfmrTbl string, uriModuleNm string) []string { + var ordTblList []string + processedTbl := false + var sncMdlList []string = getYangMdlToSonicMdlList(uriModuleNm) + + for _, sonicMdlNm := range(sncMdlList) { + sonicMdlTblInfo := xDbSpecTblSeqnMap[sonicMdlNm] + for _, ordTblNm := range(sonicMdlTblInfo.OrdTbl) { + if xfmrTbl == ordTblNm { + xfmrLogInfo("Found sonic module(%v) whose ordered table list contains table %v", sonicMdlNm, xfmrTbl) + ordTblList = sonicMdlTblInfo.OrdTbl + processedTbl = true + break + } + } + if processedTbl { + break + } + } + return ordTblList + } + +func GetXfmrOrdTblList(xfmrTbl string) []string { + /* get the table hierarchy read from json file */ + var ordTblList []string + if _, ok := sonicOrdTblListMap[xfmrTbl]; ok { + ordTblList = sonicOrdTblListMap[xfmrTbl] + } + return ordTblList +} + +func GetTablesToWatch(xfmrTblList []string, uriModuleNm string) []string { + var depTblList []string + depTblMap := make(map[string]bool) //create to avoid duplicates in depTblList, serves as a Set + processedTbl := false + var sncMdlList []string + var lXfmrTblList []string + + sncMdlList = getYangMdlToSonicMdlList(uriModuleNm) + + // remove duplicates from incoming list of tables + xfmrTblMap := make(map[string]bool) //create to avoid duplicates in xfmrTblList + for _, xfmrTblNm :=range(xfmrTblList) { + xfmrTblMap[xfmrTblNm] = true + } + for xfmrTblNm := range(xfmrTblMap) { + lXfmrTblList = append(lXfmrTblList, xfmrTblNm) + } + + for _, xfmrTbl := range(lXfmrTblList) { + processedTbl = false + //can be optimized if there is a way to know all sonic modules, a given OC-Yang spans over + for _, sonicMdlNm := range(sncMdlList) { + sonicMdlTblInfo := xDbSpecTblSeqnMap[sonicMdlNm] + for _, ordTblNm := range(sonicMdlTblInfo.OrdTbl) { + if xfmrTbl == ordTblNm { + xfmrLogInfo("Found sonic module(%v) whose ordered table list contains table %v", sonicMdlNm, xfmrTbl) + ldepTblList := sonicMdlTblInfo.DepTbl[xfmrTbl] + for _, depTblNm := range(ldepTblList) { + depTblMap[depTblNm] = true + } + //assumption that a table belongs to only one sonic module + processedTbl = true + break + } + } + if processedTbl { + break + } + } + if !processedTbl { + depTblMap[xfmrTbl] = false + } + } + for depTbl := range(depTblMap) { + depTblList = append(depTblList, depTbl) + } + return depTblList +} + +func CallRpcMethod(path string, body []byte, dbs [db.MaxDB]*db.DB) ([]byte, error) { + var err error + var ret []byte + var data []reflect.Value + var rpcFunc = "" + + // TODO - check module name + if isSonicYang(path) { + rpcName := strings.Split(path, ":") + if dbXpathData, ok := xDbSpecMap[rpcName[1]]; ok { + rpcFunc = dbXpathData.rpcFunc + } + } else { + if xpathData, ok := xYangSpecMap[path]; ok { + rpcFunc = xpathData.rpcFunc + } + } + + if rpcFunc != "" { + xfmrLogInfo("RPC callback invoked (%v) \r\n", rpcFunc) + data, err = XlateFuncCall(rpcFunc, body, dbs) + if err != nil { + return nil, err + } + ret = data[0].Interface().([]byte) + if !data[1].IsNil() { + err = data[1].Interface().(error) + } + } else { + log.Warning("Not supported RPC", path) + err = tlerr.NotSupported("Not supported RPC") + } + return ret, err +} + +func AddModelCpbltInfo() map[string]*mdlInfo { + return xMdlCpbltMap +} + +func xfmrSubscSubtreeHandler(inParams XfmrSubscInParams, xfmrFuncNm string) (XfmrSubscOutParams, error) { + var retVal XfmrSubscOutParams + retVal.dbDataMap = nil + retVal.needCache = false + retVal.onChange = false + retVal.nOpts = nil + retVal.isVirtualTbl = false + + xfmrLogInfo("Received inParams %v Subscribe Subtree function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall("Subscribe_" + xfmrFuncNm, inParams) + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == SUBSC_SBT_XFMR_RET_ARGS { + // subtree xfmr returns err as second value in return data list from .Call() + if ret[SUBSC_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[SUBSC_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Subscribe Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[SUBSC_SBT_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[SUBSC_SBT_XFMR_RET_VAL_INDX].Interface().(XfmrSubscOutParams) + } + } + return retVal, err +} + +func XlateTranslateSubscribe(path string, dbs [db.MaxDB]*db.DB, txCache interface{}) (XfmrTranslateSubscribeInfo, error) { + xfmrLogInfo("Received subcription path : %v", path) + var err error + var subscribe_result XfmrTranslateSubscribeInfo + subscribe_result.DbDataMap = make(RedisDbMap) + subscribe_result.PType = Sample + subscribe_result.MinInterval = 0 + subscribe_result.OnChange = false + subscribe_result.NeedCache = true + + for { + done := true + xpath, _, predc_err := XfmrRemoveXPATHPredicates(path) + if predc_err != nil { + log.Warningf("cannot convert request Uri to yang xpath - %v, %v", path, predc_err) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + xpathData, ok := xYangSpecMap[xpath] + if ((!ok) || (xpathData == nil)) { + log.Warningf("xYangSpecMap data not found for xpath : %v", xpath) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + if (xpathData.subscribePref == nil || ((xpathData.subscribePref != nil) &&(len(strings.TrimSpace(*xpathData.subscribePref)) == 0))) { + subscribe_result.PType = Sample + } else { + if *xpathData.subscribePref == "onchange" { + subscribe_result.PType = OnChange + } else { + subscribe_result.PType = Sample + } + } + subscribe_result.MinInterval = xpathData.subscribeMinIntvl + + if xpathData.subscribeOnChg == XFMR_DISABLE { + xfmrLogInfo("Susbcribe OnChange disabled for request Uri - %v", path) + subscribe_result.PType = Sample + subscribe_result.DbDataMap = nil + //err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + //request uri should be terminal yang object for onChange to be supported + if xpathData.hasNonTerminalNode { + xfmrLogInfo("Susbcribe request Uri is not a terminal yang object - %v", path) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + /*request uri is a key-leaf directly under the list + eg. /openconfig-xyz:xyz/listA[key=value]/key + /openconfig-xyz:xyz/listA[key_1=value][key_2=value]/key_1 + */ + if xpathData.isKey { + xfmrLogInfo("Susbcribe request Uri is not a terminal yang object - %v", path) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + xpath_dbno := xpathData.dbIndex + retData, xPathKeyExtractErr := xpathKeyExtract(dbs[xpath_dbno], nil, SUBSCRIBE, path, path, nil, nil, txCache, nil) + if ((len(xpathData.xfmrFunc) == 0) && ((xPathKeyExtractErr != nil) || ((len(strings.TrimSpace(retData.dbKey)) == 0) || (len(strings.TrimSpace(retData.tableName)) == 0)))) { + log.Warning("Error while extracting DB table/key for uri", path, "error - ", xPathKeyExtractErr) + err = xPathKeyExtractErr + break + } + if (len(xpathData.xfmrFunc) > 0) { //subtree + var inParams XfmrSubscInParams + inParams.uri = path + inParams.dbDataMap = subscribe_result.DbDataMap + inParams.dbs = dbs + inParams.subscProc = TRANSLATE_SUBSCRIBE + st_result, st_err := xfmrSubscSubtreeHandler(inParams, xpathData.xfmrFunc) + if st_err != nil { + err = st_err + break + } + subscribe_result.OnChange = st_result.onChange + xfmrLogInfo("Subtree subcribe on change %v", subscribe_result.OnChange) + if subscribe_result.OnChange { + if st_result.dbDataMap != nil { + subscribe_result.DbDataMap = st_result.dbDataMap + xfmrLogInfo("Subtree subcribe dbData %v", subscribe_result.DbDataMap) + } + subscribe_result.NeedCache = st_result.needCache + xfmrLogInfo("Subtree subcribe need Cache %v", subscribe_result.NeedCache) + } else { + subscribe_result.DbDataMap = nil + } + if st_result.nOpts != nil { + subscribe_result.PType = st_result.nOpts.pType + xfmrLogInfo("Subtree subcribe pType %v", subscribe_result.PType) + subscribe_result.MinInterval = st_result.nOpts.mInterval + xfmrLogInfo("Subtree subcribe min interval %v", subscribe_result.MinInterval) + } + } else { + subscribe_result.OnChange = true + subscribe_result.DbDataMap[xpath_dbno] = map[string]map[string]db.Value{retData.tableName: {retData.dbKey: {}}} + } + if done { + break + } + } // end of infinite for + + return subscribe_result, err + +} + +func IsTerminalNode(uri string) (bool, error) { + xpath, _, err := XfmrRemoveXPATHPredicates(uri) + if xpathData, ok := xYangSpecMap[xpath]; ok { + if !xpathData.hasNonTerminalNode { + return true, nil + } + } else { + log.Warningf("xYangSpecMap data not found for xpath : %v", xpath) + errStr := "xYangSpecMap data not found for xpath." + err = tlerr.InternalError{Format: errStr} + } + + return false, err +} + +func IsLeafNode(uri string) bool { + result := false + xpath, _, err := XfmrRemoveXPATHPredicates(uri) + if err != nil { + log.Warningf("For uri - %v, couldn't convert to xpath - %v", uri, err) + return result + } + xfmrLogInfoAll("received xpath - %v", xpath) + if xpathData, ok := xYangSpecMap[xpath]; ok { + if yangTypeGet(xpathData.yangEntry) == YANG_LEAF { + result = true + } + } else { + errStr := "xYangSpecMap data not found for xpath - " + xpath + log.Warning(errStr) + } + return result +} diff --git a/translib/transformer/xlate_cascade_del.go b/translib/transformer/xlate_cascade_del.go new file mode 100644 index 000000000000..1a5be4a3a3c7 --- /dev/null +++ b/translib/transformer/xlate_cascade_del.go @@ -0,0 +1,151 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + log "github.com/golang/glog" + "strings" + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/cvl" + "errors" + "fmt" +) + +func getDbTblCbkName(tableName string) string { + return tableName + "_cascade_cfg_hdl" +} + +func xfmrDbTblCbkHandler (inParams XfmrDbTblCbkParams, tableName string) error { + xfmrLogInfoAll("Received inParams %v xfmrDbTblCbkHandler function name %v", inParams, tableName) + + ret, err := XlateFuncCall(getDbTblCbkName(tableName), inParams) + if err != nil { + xfmrLogInfo("xfmrDbTblCbkHandler: %v failed.", getDbTblCbkName(tableName)) + return err + } + if ((ret != nil) && (len(ret)>0)) { + if ret[0].Interface() != nil { + err = ret[0].Interface().(error) + if err != nil { + log.Warningf("Table callback method i(%v) returned error - %v.", getDbTblCbkName(tableName), err) + } + } + } + + return err +} + + + +func handleCascadeDelete(d *db.DB, dbDataMap map[int]map[db.DBNum]map[string]map[string]db.Value, cascadeDelTbl [] string) error { + xfmrLogInfo("handleCascadeDelete : %v, cascadeDelTbl : %v.", dbDataMap, cascadeDelTbl) + + var err error + cvlSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + xfmrLogInfo("handleCascadeDelete : cvl.ValidationSessOpen failed.") + err = fmt.Errorf("%v", "cvl.ValidationSessOpen failed") + return err + } + defer cvl.ValidationSessClose(cvlSess) + + for operIndex, redisMap := range dbDataMap { + if operIndex != DELETE { + continue + } + for dbIndex, dbMap := range redisMap { + if dbIndex != db.ConfigDB { + continue + } + for tblIndex, tblMap := range dbMap { + if !contains(cascadeDelTbl, tblIndex) { + continue + } else { + for key, entry := range tblMap { + // need to generate key based on the db type as of now just considering configdb + // and using "|" as tablename and key seperator + depKey := tblIndex + "|" + key + depList := cvlSess.GetDepDataForDelete(depKey) + xfmrLogInfo("handleCascadeDelete : depKey : %v, depList- %v, entry : %v", depKey, depList, entry) + for depIndex, depEntry := range depList { + for depEntkey, depEntkeyInst := range depEntry.Entry { + depEntkeyList := strings.SplitN(depEntkey, "|", 2) + cbkHdlName := depEntkeyList[0] + "_cascade_cfg_hdl" + if IsXlateFuncBinded(cbkHdlName) { + //handle callback for table call Table Call back method and consolidate the data + inParams := formXfmrDbTblCbkParams(d, DELETE, depEntry.RefKey, depEntkeyList[0], depEntkeyList[1], depEntkeyInst, dbDataMap[DELETE]) + xfmrLogInfo("handleCascadeDelete CBKHDL present depIndex %v, inParams : %v ", depIndex, inParams) + err = xfmrDbTblCbkHandler(inParams, depEntkeyList[0]) + if err == nil { + for operIdx, operMap := range inParams.delDepDataMap { + if _, ok := dbDataMap[operIdx]; !ok { + dbDataMap[operIdx] = make(map[db.DBNum]map[string]map[string]db.Value) + } + for dbIndx, dbMap := range *operMap { + if _, ok := dbDataMap[operIdx][dbIndx]; !ok { + dbDataMap[operIdx][dbIndx] = make(map[string]map[string]db.Value) + } + mapMerge(dbDataMap[operIdx][dbIndx], dbMap, operIdx) + } + } + } else { + xfmrLogInfo("handleCascadeDelete - xfmrDbTblCbkHandler failed.") + return errors.New("xfmrDbTblCbkHandler failed for table: " + depEntkeyList[0] + ", Key: " + depEntkeyList[1]) + } + } else { + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]] = make(map[string]db.Value) + } + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] = db.Value{Field: make(map[string]string)} + } + + if len(depEntkeyInst) > 0 { + for depEntAttr, depEntAttrInst := range depEntkeyInst { + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]].Field[depEntAttr]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]].Field[depEntAttr] = "" + } + + if len(depEntAttrInst) > 0 { + val := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] + if strings.HasSuffix(depEntAttr, "@") { + valList := val.GetList(depEntAttr) + if !contains(valList, depEntAttrInst) { + valList = append(valList, depEntAttrInst) + val.SetList(depEntAttr, valList) + } + } else { + val.Set(depEntAttr, depEntAttrInst) + } + + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] = val + } + } + } + } + } + } + } + } + } + } + } + return nil +} + diff --git a/translib/transformer/xlate_datastructs.go b/translib/transformer/xlate_datastructs.go new file mode 100644 index 000000000000..052efd115b94 --- /dev/null +++ b/translib/transformer/xlate_datastructs.go @@ -0,0 +1,109 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/openconfig/ygot/ygot" + "regexp" +) + +var rgpIpv6, rgpMac, rgpIsMac, rgpKeyExtract, rgpSncKeyExtract *regexp.Regexp + +type tblKeyCache struct { + dbKey string + dbTblList []string +} + +type KeySpec struct { + DbNum db.DBNum + Ts db.TableSpec + Key db.Key + Child []KeySpec + IgnoreParentKey bool +} + +type NotificationType int +const ( + Sample NotificationType = iota + OnChange +) + +type XfmrTranslateSubscribeInfo struct { + DbDataMap RedisDbMap + MinInterval int + NeedCache bool + PType NotificationType + OnChange bool +} + +type xpathTblKeyExtractRet struct { + xpath string + tableName string + dbKey string + isVirtualTbl bool +} + +type xlateFromDbParams struct { + d *db.DB //current db + dbs [db.MaxDB]*db.DB + curDb db.DBNum + ygRoot *ygot.GoStruct + uri string + requestUri string //original uri using which a curl/NBI request is made + oper int + dbDataMap *map[db.DBNum]map[string]map[string]db.Value + // subOpDataMap map[int]*RedisDbMap // used to add an in-flight data with a sub-op + // param interface{} + txCache interface{} + // skipOrdTblChk *bool + // pCascadeDelTbl *[] string //used to populate list of tables needed cascade delete by subtree overloaded methods + xpath string //curr uri xpath + tbl string + tblKey string + resultMap map[string]interface{} + validate bool + xfmrDbTblKeyCache map[string]tblKeyCache + dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool +} + +type xlateToParams struct { + d *db.DB + ygRoot *ygot.GoStruct + oper int + uri string + requestUri string + xpath string + keyName string + jsonData interface{} + resultMap map[int]RedisDbMap + result map[string]map[string]db.Value + txCache interface{} + tblXpathMap map[string]map[string]map[string]bool + subOpDataMap map[int]*RedisDbMap + pCascadeDelTbl *[]string + xfmrErr *error + name string + value interface{} + tableName string + yangDefValMap map[string]map[string]db.Value + yangAuxValMap map[string]map[string]db.Value + xfmrDbTblKeyCache map[string]tblKeyCache + dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool +} diff --git a/translib/transformer/xlate_del_to_db.go b/translib/transformer/xlate_del_to_db.go new file mode 100644 index 000000000000..8da6b79bb01b --- /dev/null +++ b/translib/transformer/xlate_del_to_db.go @@ -0,0 +1,888 @@ + //////////////////////////////////////////////////////////////////////////////// + // // + // Copyright 2019 Dell, Inc. // + // // + // Licensed under the Apache License, Version 2.0 (the "License"); // + // you may not use this file except in compliance with the License. // + // You may obtain a copy of the License at // + // // + // http://www.apache.org/licenses/LICENSE-2.0 // + // // + // Unless required by applicable law or agreed to in writing, software // + // distributed under the License is distributed on an "AS IS" BASIS, // + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // + // See the License for the specific language governing permissions and // + // limitations under the License. // + // // + //////////////////////////////////////////////////////////////////////////////// + +package transformer +import ( + "errors" + "strings" + "fmt" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/openconfig/goyang/pkg/yang" + "github.com/openconfig/ygot/ygot" + log "github.com/golang/glog" +) + +func tblKeyDataGet(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) ([]string, bool, error) { + var err error + var dbs [db.MaxDB]*db.DB + var tblList []string + dbs[cdb] = xlateParams.d + isVirtualTbl := false + + xfmrLogInfoAll("Get table data for (\"%v\")", xlateParams.uri) + if (xYangSpecMap[xlateParams.xpath].tableName != nil) && (len(*xYangSpecMap[xlateParams.xpath].tableName) > 0) { + tblList = append(tblList, *xYangSpecMap[xlateParams.xpath].tableName) + } else if xYangSpecMap[xlateParams.xpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[xlateParams.xpath].xfmrTbl + if len(xfmrTblFunc) > 0 { + inParams := formXfmrInputRequest(xlateParams.d, dbs, cdb, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, xlateParams.keyName, dbDataMap, nil, nil, xlateParams.txCache) + tblList, err = xfmrTblHandlerFunc(xfmrTblFunc, inParams, xlateParams.xfmrDbTblKeyCache) + if err != nil { + return tblList, isVirtualTbl, err + } + if inParams.isVirtualTbl != nil { + isVirtualTbl = *(inParams.isVirtualTbl) + } + } + } + tbl := xlateParams.tableName + if tbl != "" { + if !contains(tblList, tbl) { + tblList = append(tblList, tbl) + } + } + return tblList, isVirtualTbl, err +} + +func subTreeXfmrDelDataGet(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum, spec *yangXpathInfo, chldSpec *yangXpathInfo, subTreeResMap *map[string]map[string]db.Value) error { + var dbs [db.MaxDB]*db.DB + dbs[cdb] = xlateParams.d + + xfmrLogInfoAll("Handle subtree for (\"%v\")", xlateParams.uri) + if (len(chldSpec.xfmrFunc) > 0) { + if ((len(spec.xfmrFunc) == 0) || ((len(spec.xfmrFunc) > 0) && + (spec.xfmrFunc != chldSpec.xfmrFunc))) { + inParams := formXfmrInputRequest(xlateParams.d, dbs, cdb, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, "", + dbDataMap, xlateParams.subOpDataMap, nil, xlateParams.txCache) + retMap, err := xfmrHandler(inParams, chldSpec.xfmrFunc) + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", chldSpec.xfmrFunc, err) + return err + } + mapCopy(*subTreeResMap, retMap) + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } + } + } + return nil +} + +func yangListDelData(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subTreeResMap *map[string]map[string]db.Value, isFirstCall bool) error { + var err, perr error + var dbs [db.MaxDB]*db.DB + var tblList []string + xfmrLogInfoAll("yangListDelData Received xlateParams - %v \n dbDataMap - %v\n subTreeResMap - %v\n isFirstCall - %v", xlateParams, dbDataMap, subTreeResMap, isFirstCall) + fillFields := false + removedFillFields := false + virtualTbl := false + tblOwner := true + keyName := xlateParams.keyName + parentTbl := "" + parentKey := "" + + spec, xpathOk := xYangSpecMap[xlateParams.xpath] + if !xpathOk { + return err + } + if ((spec.yangEntry != nil) && (spec.yangEntry.ReadOnly())) { + xfmrLogInfoAll("For Uri - %v skip delete processing since its a Read Only node", xlateParams.uri) + return err + } + cdb := spec.dbIndex + dbs[cdb] = xlateParams.d + dbOpts := getDBOptions(cdb) + separator := dbOpts.KeySeparator + + if !isFirstCall { + xpathKeyExtRet, err := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + if err != nil { + xfmrLogInfoAll("Received error from xpathKeyExtract for uri : %v, error: %v", xlateParams.uri, err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if xpathKeyExtRet.isVirtualTbl { + virtualTbl = true + } + keyName = xpathKeyExtRet.dbKey + xlateParams.tableName = xpathKeyExtRet.tableName + xlateParams.keyName = keyName + } + + tblList, virtualTbl, err = tblKeyDataGet(xlateParams, dbDataMap, cdb) + if err != nil { + return err + } + + xfmrLogInfoAll("tblList(%v), tbl(%v), key(%v) for uri (\"%v\")", tblList, xlateParams.tableName, xlateParams.keyName, xlateParams.uri) + for _, tbl := range(tblList) { + curDbDataMap, ferr := fillDbDataMapForTbl(xlateParams.uri, xlateParams.xpath, tbl, keyName, cdb, dbs, xlateParams.dbTblKeyGetCache) + if ((ferr == nil) && len(curDbDataMap) > 0) { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + } + } + + // Not required to process parent and current table as subtree is already invoked before we get here + // We only need to traverse nested subtrees here + if isFirstCall && len(spec.xfmrFunc) == 0 { + parentUri := parentUriGet(xlateParams.uri) + xpathKeyExtRet , xerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, parentUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + parentTbl = xpathKeyExtRet.tableName + parentKey = xpathKeyExtRet.dbKey + perr = xerr + xfmrLogInfoAll("Parent Uri - %v, ParentTbl - %v, parentKey - %v", parentUri, parentTbl, parentKey) + } + + for _, tbl := range(tblList) { + tblData, ok := (*dbDataMap)[cdb][tbl] + xfmrLogInfoAll("Process Tbl - %v", tbl) + var curTbl, curKey string + var cerr error + if ok { + for dbKey := range tblData { + xfmrLogInfoAll("Process Tbl - %v with dbKey - %v", tbl, dbKey) + _, curUri, kerr := dbKeyToYangDataConvert(xlateParams.uri, xlateParams.requestUri, xlateParams.xpath, tbl, dbDataMap, dbKey, separator, xlateParams.txCache) + if kerr != nil { + continue + } + if spec.virtualTbl != nil && *spec.virtualTbl { + virtualTbl = true + } + + // Not required to check for table inheritence case here as we have a subtree and subtree is already processed before we get here + // We only need to traverse nested subtrees here + if len(spec.xfmrFunc) == 0 { + + xpathKeyExtRet, xerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + curKey = xpathKeyExtRet.dbKey + curTbl = xpathKeyExtRet.tableName + cerr = xerr + xfmrLogInfoAll("Current Uri - %v, CurrentTbl - %v, CurrentKey - %v", curUri, curTbl, curKey) + + if dbKey != curKey { + continue + } + if isFirstCall { + if perr == nil && cerr == nil { + if len(curTbl) > 0 && parentTbl != curTbl { + /* Non-inhertited table case */ + xfmrLogInfoAll("Non-inhertaed table case, uri - %v", curUri) + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + } + } else if len(curTbl) > 0 { + /* Inhertited table case */ + xfmrLogInfoAll("Inherited table case, uri - %v", curUri) + if len(parentKey) > 0 { + if parentKey == curKey { // List within list or List within container, where container map to entire table + xfmrLogInfoAll("Parent key is same as current key") + xfmrLogInfoAll("Request from NBI is at same level as that of current list - %v", curUri) + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false // since query is at this level, this will make sure to add instance to result + } + /* Fill only fields */ + fillFields = true + } else { /*same table but different keys */ + xfmrLogInfoAll("Inherited table but parent key is NOT same as current key") + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + } + } + } else { + /*same table but no parent-key exists, parent must be a container wth just tableNm annot with no keyXfmr/Nm */ + xfmrLogInfoAll("Inherited table but no parent key available") + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + + } + } + } else { // len(curTbl) = 0 + log.Warning("No table found for Uri - %v ", curUri) + } + } + } else { + /* if table instance already filled and there are no feilds present then it's instance level delete. + If fields present then its fields delet and not instance delete + */ + if tblData, tblDataOk := xlateParams.result[curTbl]; tblDataOk { + if fieldMap, fieldMapOk := tblData[curKey]; fieldMapOk { + xfmrLogInfoAll("Found table instance same as that of requestUri") + if len(fieldMap.Field) > 0 { + /* Fill only fields */ + xfmrLogInfoAll("Found table instance same as that of requestUri with fields.") + fillFields = true + } + } + } + + }// end of if isFirstCall + } // end if !subtree case + xfmrLogInfoAll("For uri - %v , table-owner - %v, fillFields - %v", curUri, tblOwner, fillFields) + if fillFields || spec.hasChildSubTree || isFirstCall { + for yangChldName := range spec.yangEntry.Dir { + chldXpath := xlateParams.xpath+"/"+yangChldName + chldUri := curUri+"/"+yangChldName + chldSpec, ok := xYangSpecMap[chldXpath] + if (ok && ((chldSpec.yangEntry != nil) && (!chldSpec.yangEntry.ReadOnly()))) { + chldYangType := chldSpec.yangDataType + curXlateParams := xlateParams + curXlateParams.uri = chldUri + curXlateParams.xpath = chldXpath + curXlateParams.tableName = "" + curXlateParams.keyName = "" + + if (chldSpec.dbIndex == db.ConfigDB) && (len(chldSpec.xfmrFunc) > 0) { + err = subTreeXfmrDelDataGet(curXlateParams, dbDataMap, cdb, spec, chldSpec, subTreeResMap) + if err != nil { + return err + } + } + if chldYangType == YANG_CONTAINER { + err = yangContainerDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if chldYangType == YANG_LIST { + err = yangListDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if (chldSpec.dbIndex == db.ConfigDB) && ((chldYangType == YANG_LEAF) || (chldYangType == YANG_LEAF_LIST)) && !virtualTbl { + if len(curTbl) == 0 { + continue + } + if len(curKey) == 0 { //at list level key should always be there + xfmrLogInfoAll("No key avaialble for uri - %v", curUri) + continue + } + if chldYangType == YANG_LEAF && chldSpec.isKey { + if isFirstCall { + if !tblOwner { //add dummy field to identify when to fill fields only at children traversal + dataToDBMapAdd(curTbl, curKey, curXlateParams.result, "FillFields", "true") + } else { + dataToDBMapAdd(curTbl, curKey, curXlateParams.result, "", "") + } + } + } else if fillFields { + //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + curXlateParams.uri = xlateParams.uri + curXlateParams.name = chldSpec.yangEntry.Name + curXlateParams.tableName = curTbl + curXlateParams.keyName = curKey + err = mapFillDataUtil(curXlateParams) + if err != nil { + xfmrLogInfoAll("Error received (\"%v\")", err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if !removedFillFields { + if fieldMap, ok := curXlateParams.result[curTbl][curKey]; ok { + if len(fieldMap.Field) > 1 { + delete(curXlateParams.result[curTbl][curKey].Field, "FillFields") + removedFillFields = true + } else if len(fieldMap.Field) == 1 { + if _, ok := curXlateParams.result[curTbl][curKey].Field["FillFields"]; !ok { + removedFillFields = true + } + } + } + } + } + } // end of Leaf case + } // if rw + } // end of curUri children traversal loop + } // Child Subtree or fill fields + } // end of for dbKey loop + } // end of tbl in dbDataMap +} // end of for tbl loop +return err +} + +func yangContainerDelData(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subTreeResMap *map[string]map[string]db.Value, isFirstCall bool) error { + var err error + var dbs [db.MaxDB]*db.DB + spec, ok := xYangSpecMap[xlateParams.xpath] + cdb := spec.dbIndex + dbs[cdb] = xlateParams.d + removedFillFields := false + var curTbl, curKey string + + if !ok { + return err + } + + if (ok && (spec.yangEntry != nil) && (spec.yangEntry.ReadOnly())) { + return err + } + + xfmrLogInfoAll("Traverse container for DELETE at uri (\"%v\")", xlateParams.uri) + + fillFields := false + + // Not required to process parent and current table as subtree is already invoked before we get here + // We only need to traverse nested subtrees here + if len(spec.xfmrFunc) == 0 { + xpathKeyExtRet, cerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + curKey = xpathKeyExtRet.dbKey + curTbl = xpathKeyExtRet.tableName + if cerr != nil { + log.Warningf("Received xpathKeyExtract error for uri: %v : err %v", xlateParams.uri, cerr) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + xfmrLogInfoAll("Error received (\"%v\"), ecode :%v", cerr, ecode) + if ecode { + return cerr + } + } + } + + if isFirstCall { + parentUri := parentUriGet(xlateParams.uri) + parentXpathKeyExtRet, perr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, parentUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + parentTbl := parentXpathKeyExtRet.tableName + parentKey := parentXpathKeyExtRet.dbKey + if perr == nil && cerr == nil && len(curTbl) > 0 { + if len(curKey) > 0 { + xfmrLogInfoAll("DELETE handling at Container parentTbl %v, curTbl %v, curKey %v", parentTbl, curTbl, curKey) + if (parentTbl != curTbl) { + // Non inhertited table + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + // Fill fields only + xfmrLogInfoAll("DELETE handling at Container Non inhertited table and not table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } else if (spec.keyName != nil && len(*spec.keyName) > 0) || len(spec.xfmrKey) > 0 { + // Table owner && Key transformer present. Fill table instance + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } else { + // Fallback case. Ideally should not enter here + fillFields = true + } + } else { + if curKey != parentKey { + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + xfmrLogInfoAll("DELETE handling at Container inhertited table and not table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } else { + // Instance delete + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } + } else { + // if Instance already filled do not fill fields + xfmrLogInfoAll("DELETE handling at Container Inherited table") + //Fill fields only + if len(curTbl) > 0 && len(curKey) > 0 { + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } + } + } + } else { + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + // Fill fields only + xfmrLogInfoAll("DELETE handling at Container Non inhertited table and not table Owner No Key available. table: %v, key: %v", curTbl, curKey) + } else { + // Table owner && Key transformer present. Fill table instance + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner. No Key Delete complete TABLE : %v", curTbl) + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } + } + } else { + xfmrLogInfoAll("perr: %v cerr: %v curTbl: %v, curKey: %v", perr, cerr, curTbl, curKey) + } + } else { + // Inherited Table. We always expect the curTbl entry in xlateParams.result + // if Instance already filled do not fill fields + xfmrLogInfoAll("DELETE handling at Container Inherited table curTbl: %v, curKey %v", curTbl, curKey) + if tblMap, ok := xlateParams.result[curTbl]; ok { + if fieldMap, ok := tblMap[curKey]; ok { + if len(fieldMap.Field) == 0 { + xfmrLogInfoAll("Inhertited table & Instance delete case. Skip fields fill") + } else { + xfmrLogInfoAll("Inhertited table & fields fill for table :%v", curTbl) + fillFields = true + } + } + } + } + + } + xfmrLogInfoAll("uri %v fillFields %v, hasChildSubtree %v, isFirstCall %v", xlateParams.uri, fillFields, spec.hasChildSubTree, isFirstCall) + + if (fillFields || spec.hasChildSubTree || isFirstCall) { + for yangChldName := range spec.yangEntry.Dir { + chldXpath := xlateParams.xpath+"/"+yangChldName + chldUri := xlateParams.uri+"/"+yangChldName + chldSpec, ok := xYangSpecMap[chldXpath] + if (ok && ((chldSpec.yangEntry != nil) && (!chldSpec.yangEntry.ReadOnly()))) { + chldYangType := chldSpec.yangDataType + curXlateParams := xlateParams + curXlateParams.uri = chldUri + curXlateParams.xpath = chldXpath + curXlateParams.tableName = curTbl + curXlateParams.keyName = curKey + + if (chldSpec.dbIndex == db.ConfigDB) && (len(chldSpec.xfmrFunc) > 0) { + err = subTreeXfmrDelDataGet(curXlateParams, dbDataMap, cdb, spec, chldSpec, subTreeResMap) + if err != nil { + return err + } + } + if chldYangType == YANG_CONTAINER { + err = yangContainerDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if chldYangType == YANG_LIST { + err = yangListDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if (chldSpec.dbIndex == db.ConfigDB) && (chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST) && fillFields { + //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + curXlateParams.uri = xlateParams.uri + curXlateParams.name = chldSpec.yangEntry.Name + err = mapFillDataUtil(curXlateParams) + if err != nil { + xfmrLogInfoAll("Error received during leaf fill (\"%v\")", err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if !removedFillFields { + if fieldMap, ok := curXlateParams.result[curTbl][curKey]; ok { + if len(fieldMap.Field) > 1 { + delete(curXlateParams.result[curTbl][curKey].Field, "FillFields") + removedFillFields = true + } else if len(fieldMap.Field) == 1 { + if _, ok := curXlateParams.result[curTbl][curKey].Field["FillFields"]; !ok { + removedFillFields = true + } + } + } + } + } else { + xfmrLogInfoAll("%v", "Instance Fill case. Have filled the result table with table and key") + } + } + } + } + return err +} + +func allChildTblGetToDelete(xlateParams xlateToParams) (map[string]map[string]db.Value, error) { + var err error + subTreeResMap := make(map[string]map[string]db.Value) + xpath, _, _ := XfmrRemoveXPATHPredicates(xlateParams.requestUri) + spec, ok := xYangSpecMap[xpath] + isFirstCall := true + if !ok { + errStr := "Xpath not found in spec-map:" + xpath + return subTreeResMap, errors.New(errStr) + } + + dbDataMap := make(RedisDbMap) + for i := db.ApplDB; i < db.MaxDB; i++ { + dbDataMap[i] = make(map[string]map[string]db.Value) + } + + if len(spec.xfmrFunc) > 0 { + // Subtree is already invoked before we get here + // Not required to process parent and current tables + isFirstCall = false + } + + xfmrLogInfoAll("Req-uri (\"%v\") to traverse for delete", xlateParams.requestUri) + if ok && spec.yangEntry != nil { + xlateParams.uri = xlateParams.requestUri + xlateParams.xpath = xpath + if (spec.yangDataType == YANG_LIST) { + err = yangListDelData(xlateParams, &dbDataMap, &subTreeResMap, isFirstCall) + return subTreeResMap, err + } else if (spec.yangDataType == YANG_CONTAINER) { + err = yangContainerDelData(xlateParams, &dbDataMap, &subTreeResMap, isFirstCall) + } + } + return subTreeResMap, err +} + +/* Get the db table, key and field name for the incoming delete request */ +func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, jsonData interface{}, resultMap map[int]map[db.DBNum]map[string]map[string]db.Value, txCache interface{}, skipOrdTbl *bool) error { + var err error + var result = make(map[string]map[string]db.Value) + subOpDataMap := make(map[int]*RedisDbMap) + var xfmrErr error + *skipOrdTbl = false + var cascadeDelTbl []string + + /* Check if the parent table exists for RFC compliance */ + var exists bool + var dbs [db.MaxDB]*db.DB + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(d, dbs, ygRoot, oper, uri, nil, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + if exists { + // Special case when we delete at container that does exist. Not required to do translation. Do not return error either. + return nil + } + log.Warningf("Cannot perform Operation %v on uri %v due to - %v", oper, uri, err) + return err + } + if !exists { + errStr := fmt.Sprintf("Parent table does not exist for uri(%v)", uri) + return tlerr.InternalError{Format: errStr} + } + for i := 0; i < MAXOPER; i++ { + resultMap[i] = make(map[db.DBNum]map[string]map[string]db.Value) + } + + if isSonicYang(uri) { + xpathPrefix, keyName, tableName := sonicXpathKeyExtract(uri) + xfmrLogInfo("Delete req: uri(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", uri, keyName, xpathPrefix, tableName) + resultMap[oper][db.ConfigDB] = result + xlateToData := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpathPrefix, keyName, jsonData, resultMap, result, txCache, nil, subOpDataMap, &cascadeDelTbl, &xfmrErr, "","",tableName) + err = sonicYangReqToDbMapDelete(xlateToData) + if err != nil { + return err + } + } else { + xpathKeyExtRet, err := xpathKeyExtract(d, ygRoot, oper, uri, requestUri, nil, subOpDataMap, txCache, nil) + if err != nil { + return err + } + xfmrLogInfo("Delete req: uri(\"%v\"), key(\"%v\"), xpath(\"%v\"), tableName(\"%v\").", uri, xpathKeyExtRet.dbKey, xpathKeyExtRet.xpath, xpathKeyExtRet.tableName) + spec, ok := xYangSpecMap[xpathKeyExtRet.xpath] + if ok { + specYangType := yangTypeGet(spec.yangEntry) + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) + if modSpecInfo, specOk := xYangSpecMap[moduleNm]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + var dbs [db.MaxDB]*db.DB + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre-transformer: %v, oper: %v, subOpDataMap: %v ", + modSpecInfo.xfmrPre, oper, subOpDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return err + } + } + + if spec.cascadeDel == XFMR_ENABLE && xpathKeyExtRet.tableName != "" && xpathKeyExtRet.tableName != XFMR_NONE_STRING { + if !contains(cascadeDelTbl, xpathKeyExtRet.tableName) { + cascadeDelTbl = append(cascadeDelTbl, xpathKeyExtRet.tableName) + } + } + curXlateParams := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpathKeyExtRet.xpath, xpathKeyExtRet.dbKey, jsonData, resultMap, result, txCache, nil, subOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", xpathKeyExtRet.tableName) + curXlateParams.xfmrDbTblKeyCache = make(map[string]tblKeyCache) + if len(spec.xfmrFunc) > 0 { + var dbs [db.MaxDB]*db.DB + cdb := spec.dbIndex + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + stRetData, err := xfmrHandler(inParams, spec.xfmrFunc) + if err == nil { + mapCopy(result, stRetData) + } else { + return err + } + // TODO: Nested subtree invoke + curResult, cerr := allChildTblGetToDelete(curXlateParams) + if cerr != nil { + return cerr + } else { + mapCopy(result, curResult) + } + + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } + } else if specYangType == YANG_LEAF || specYangType == YANG_LEAF_LIST { + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + dataToDBMapAdd(xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, result, "", "") + xpath := xpathKeyExtRet.xpath + uriItemList := splitUri(strings.TrimSuffix(uri, "/")) + uriItemListLen := len(uriItemList) + var luri string + if uriItemListLen > 0 { + luri = strings.Join(uriItemList[:uriItemListLen-1], "/") //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + + } + if specYangType == YANG_LEAF { + _, ok := xYangSpecMap[xpath] + if ok && len(xYangSpecMap[xpath].defVal) > 0 { + // Do not fill def value if leaf does not map to any redis field + dbSpecXpath := xpathKeyExtRet.tableName + "/" + xYangSpecMap[xpath].fieldName + _, mapped := xDbSpecMap[dbSpecXpath] + if mapped || len(xYangSpecMap[xpath].xfmrField) > 0 { + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + curXlateParams.value = xYangSpecMap[xpath].defVal + err = mapFillDataUtil(curXlateParams) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + if len(subOpDataMap) > 0 && subOpDataMap[UPDATE] != nil { + subOperMap := subOpDataMap[UPDATE] + mapCopy((*subOperMap)[db.ConfigDB], result) + } else { + var redisMap = new(RedisDbMap) + var dbresult = make(RedisDbMap) + for i := db.ApplDB; i < db.MaxDB; i++ { + dbresult[i] = make(map[string]map[string]db.Value) + } + redisMap = &dbresult + (*redisMap)[db.ConfigDB] = result + subOpDataMap[UPDATE] = redisMap + } + } + result = make(map[string]map[string]db.Value) + } else { + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + err = mapFillDataUtil(curXlateParams) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + } + } else if specYangType == YANG_LEAF_LIST { + var fieldVal []interface{} + leafListInstVal, valErr := extractLeafListInstFromUri(uri) + if valErr == nil && leafListInstVal != "" { + fieldVal = append(fieldVal, leafListInstVal) + } + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + curXlateParams.value = fieldVal + err = mapFillDataUtil(curXlateParams) + + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + } + } else { + log.Warningf("No proper table and key information to fill result map for uri %v, table: %v, key %v", uri, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + } + } else { + xfmrLogInfoAll("Before calling allChildTblGetToDelete result: %v", curXlateParams.result) + curResult, cerr := allChildTblGetToDelete(curXlateParams) + if cerr != nil { + err = cerr + return err + } else { + mapCopy(result, curResult) + } + xfmrLogInfoAll("allChildTblGetToDelete result: %v subtree curResult: %v", result, curResult) + // Add the child tables to delete when table at request URI is not available or its complete table delete request (not specific instance) + chResult := make(map[string]map[string]db.Value) + if (len(xpathKeyExtRet.tableName) == 0 || (len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) == 0)) && len(spec.childTable) > 0 { + for _, child := range spec.childTable { + chResult[child] = make(map[string]db.Value) + } + xfmrLogInfoAll("Before adding children result: %v result with child tables: %v", result, chResult) + } + mapCopy(result, chResult) + } + + _, ok = xYangSpecMap[moduleNm] + if ok && len(xYangSpecMap[moduleNm].xfmrPost) > 0 { + xfmrLogInfo("Invoke post transformer: %v", xYangSpecMap[moduleNm].xfmrPost) + var dbs [db.MaxDB]*db.DB + var dbresult = make(RedisDbMap) + dbresult[db.ConfigDB] = result + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", &dbresult, subOpDataMap, nil, txCache) + result, err = postXfmrHandlerFunc(xYangSpecMap[moduleNm].xfmrPost, inParams) + if err != nil { + return err + } + if inParams.skipOrdTblChk != nil { + *skipOrdTbl = *(inParams.skipOrdTblChk) + xfmrLogInfo("skipOrdTbl flag: %v", *skipOrdTbl) + } + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } + } + + if len(result) > 0 { + resultMap[oper][db.ConfigDB] = result + } + + if len(subOpDataMap) > 0 { + for op, data := range subOpDataMap { + if len(*data) > 0 { + for dbType, dbData := range (*data) { + if len(dbData) > 0 { + if _, ok := resultMap[op][dbType]; !ok { + resultMap[op][dbType] = make(map[string]map[string]db.Value) + } + mapCopy(resultMap[op][dbType], (*subOpDataMap[op])[dbType]) + } + } + } + } + + } + /* for container/list delete req , it should go through, even if there are any leaf default-yang-values */ + } + } // End OC yang handling + + err = dbDataXfmrHandler(resultMap) + if err != nil { + log.Warningf("Failed in dbdata-xfmr for %v", resultMap) + return err + } + if (len(cascadeDelTbl) > 0) { + cdErr := handleCascadeDelete(d, resultMap, cascadeDelTbl) + if cdErr != nil { + xfmrLogInfo("Cascade Delete Failed for cascadeDelTbl (%v), Error: (%v)", cascadeDelTbl, cdErr) + return cdErr + } + } + + printDbData(resultMap, nil, "/tmp/yangToDbDataDel.txt") + xfmrLogInfo("Delete req: uri(\"%v\") resultMap(\"%v\").", uri, resultMap) + return err +} + +func sonicYangReqToDbMapDelete(xlateParams xlateToParams) error { + var err error + if (xlateParams.tableName != "") { + // Specific table entry case + xlateParams.result[xlateParams.tableName] = make(map[string]db.Value) + isFieldReq := false + if (xlateParams.keyName != "") { + // Specific key case + var dbVal db.Value + tokens:= strings.Split(xlateParams.xpath, "/") + if tokens[SONIC_TABLE_INDEX] == xlateParams.tableName { + fieldName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + } + + if fieldName != "" { + isFieldReq = true + dbSpecField := xlateParams.tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok { + yangType := xDbSpecMap[dbSpecField].fieldType + // terminal node case + if yangType == YANG_LEAF_LIST { + dbVal.Field = make(map[string]string) + dbFldVal := "" + //check if it is a specific item in leaf-list delete + leafListInstVal, valErr := extractLeafListInstFromUri(xlateParams.requestUri) + if valErr == nil { + dbFldVal, err = unmarshalJsonToDbData(xDbSpecMap[dbSpecField].dbEntry, dbSpecField, fieldName, leafListInstVal) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", dbSpecField, err) + return err + } + } + fieldName = fieldName + "@" + dbVal.Field[fieldName] = dbFldVal + } + if yangType == YANG_LEAF { + dbVal.Field = make(map[string]string) + dbVal.Field[fieldName] = "" + } + } + } + } + xlateParams.result[xlateParams.tableName][xlateParams.keyName] = dbVal + } + if !isFieldReq { + if tblSpecInfo, ok := xDbSpecMap[xlateParams.tableName]; ok && (tblSpecInfo.cascadeDel == XFMR_ENABLE) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, xlateParams.tableName) + } + } + } else { + // Get all table entries + // If table name not available in xpath get top container name + _, ok := xDbSpecMap[xlateParams.xpath] + if ok && xDbSpecMap[xlateParams.xpath] != nil { + dbInfo := xDbSpecMap[xlateParams.xpath] + if dbInfo.fieldType == "container" { + for dir := range dbInfo.dbEntry.Dir { + if tblSpecInfo, ok := xDbSpecMap[dir]; ok && tblSpecInfo.cascadeDel == XFMR_ENABLE { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, dir) + } + if dbInfo.dbEntry.Dir[dir].Config != yang.TSFalse { + xlateParams.result[dir] = make(map[string]db.Value) + } + } + } + } + } + return nil +} + diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index 949f6c27c31d..8d26c75d28db 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -23,91 +23,18 @@ import ( "github.com/Azure/sonic-mgmt-common/translib/db" "strings" "encoding/json" - "os" "strconv" "errors" - "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "sync" "github.com/openconfig/goyang/pkg/yang" - "github.com/openconfig/ygot/ygot" - "github.com/openconfig/ygot/ytypes" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" log "github.com/golang/glog" ) type typeMapOfInterface map[string]interface{} -func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { - result := make(map[string]interface{}) - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - log.Infof("Subtree transformer function(\"%v\") invoked for yang path(\"%v\").", xYangSpecMap[xpath].xfmrFunc, xpath) - _, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - log.Infof("Failed to retrieve data for xpath(\"%v\") err(%v).", inParams.uri, err) - return result, err - } - - ocbSch, _ := ocbinds.Schema() - schRoot := ocbSch.RootSchema() - device := (*inParams.ygRoot).(*ocbinds.Device) - - path, _ := ygot.StringToPath(inParams.uri, ygot.StructuredPath, ygot.StringSlicePath) - for _, p := range path.Elem { - pathSlice := strings.Split(p.Name, ":") - p.Name = pathSlice[len(pathSlice)-1] - if len(p.Key) > 0 { - for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] - } - } - } - - nodeList, nodeErr := ytypes.GetNode(schRoot, device, path) - if nodeErr != nil { - log.Infof("Failed to get node for xpath(\"%v\") err(%v).", inParams.uri, err) - return result, err - } - node := nodeList[0].Data - nodeYgot, _ := (node).(ygot.ValidatedGoStruct) - payload, err := ygot.EmitJSON(nodeYgot, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, - Indent: " ", SkipValidation: true, - RFC7951Config: &ygot.RFC7951JSONConfig{ AppendModuleName: false, }, - }) - err = json.Unmarshal([]byte(payload), &result) - return result, err -} - -func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return nil, err - } - if ret != nil { - fldValMap := ret[0].Interface().(map[string]interface{}) - return fldValMap, nil - } else { - return nil, nil - } -} - -func validateHandlerFunc(inParams XfmrParams) (bool) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xYangSpecMap[xpath].validateFunc, inParams) - if err != nil { - return false - } - return ret[0].Interface().(bool) -} - -func xfmrTblHandlerFunc(xfmrTblFunc string, inParams XfmrParams) []string { - ret, err := XlateFuncCall(xfmrTblFunc, inParams) - if err != nil { - return []string{} - } - return ret[0].Interface().([]string) -} - +var mapCopyMutex = &sync.Mutex{} func DbValToInt(dbFldVal string, base int, size int, isUint bool) (interface{}, error) { var res interface{} @@ -124,11 +51,88 @@ func DbValToInt(dbFldVal string, base int, size int, isUint bool) (interface{}, return res, err } -func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal string) (interface{}, error) { - log.Infof("Received FieldXpath %v, yngTerminalNdDtType %v and Db field value %v to be converted to yang data-type.", fldXpath, yngTerminalNdDtType, dbFldVal) +func getLeafrefRefdYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string) (yang.TypeKind) { + if yngTerminalNdDtType == yang.Yleafref { + var entry *yang.Entry + var path string + if _, ok := xDbSpecMap[fldXpath]; ok { + path = xDbSpecMap[fldXpath].dbEntry.Type.Path + entry = xDbSpecMap[fldXpath].dbEntry + } else if _, ok := xYangSpecMap[fldXpath]; ok { + path = xYangSpecMap[fldXpath].yangEntry.Type.Path + entry = xYangSpecMap[fldXpath].yangEntry + } + path = stripAugmentedModuleNames(path) + path = path[1:] + xfmrLogInfoAll("Received path %v for FieldXpath %v", path, fldXpath) + if strings.Contains(path, "..") { + if entry != nil && len(path) > 0 { + // Referenced path within same yang file + xpath, _, err := XfmrRemoveXPATHPredicates(path) + if err != nil { + log.Warningf("error in XfmrRemoveXPATHPredicates %v", path) + return yngTerminalNdDtType + } + xpath = xpath[1:] + pathList := strings.Split(xpath, "/") + for _, x := range pathList { + if x == ".." { + entry = entry.Parent + } else { + if _,ok := entry.Dir[x]; ok { + entry = entry.Dir[x] + } + } + } + if entry != nil && entry.Type != nil { + yngTerminalNdDtType = entry.Type.Kind + xfmrLogInfoAll("yangLeaf datatype %v", yngTerminalNdDtType) + if yngTerminalNdDtType == yang.Yleafref { + leafPath := getXpathFromYangEntry(entry) + xfmrLogInfoAll("getLeafrefRefdYangType: xpath for leafref type:%v",leafPath) + return getLeafrefRefdYangType(yngTerminalNdDtType, leafPath) + } + } + } + } else if len(path) > 0 { + // Referenced path in a different yang file + xpath, _, err := XfmrRemoveXPATHPredicates(path) + if err != nil { + log.Warningf("error in XfmrRemoveXPATHPredicates %v", xpath) + return yngTerminalNdDtType + } + // Form xpath based on sonic or non sonic yang path + if strings.Contains(xpath, "sonic") { + pathList := strings.Split(xpath, "/") + xpath = pathList[SONIC_TABLE_INDEX]+ "/" + pathList[SONIC_FIELD_INDEX] + if _, ok := xDbSpecMap[xpath]; ok { + yngTerminalNdDtType = xDbSpecMap[xpath].dbEntry.Type.Kind + } + + } else { + xpath = replacePrefixWithModuleName(xpath) + if _, ok := xYangSpecMap[xpath]; ok { + yngTerminalNdDtType = xYangSpecMap[xpath].dbEntry.Type.Kind + } + } + + } + xfmrLogInfoAll("yangLeaf datatype %v", yngTerminalNdDtType) + } + return yngTerminalNdDtType +} + +func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal string) (interface{}, interface{}, error) { + xfmrLogInfoAll("Received FieldXpath %v, yngTerminalNdDtType %v and Db field value %v to be converted to yang data-type.", fldXpath, yngTerminalNdDtType, dbFldVal) var res interface{} + var resPtr interface{} var err error const INTBASE = 10 + + if yngTerminalNdDtType == yang.Yleafref { + yngTerminalNdDtType = getLeafrefRefdYangType(yngTerminalNdDtType, fldXpath) + } + switch yngTerminalNdDtType { case yang.Ynone: log.Warning("Yang node data-type is non base yang type") @@ -136,53 +140,70 @@ func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal s err = errors.New("Yang node data-type is non base yang type") case yang.Yint8: res, err = DbValToInt(dbFldVal, INTBASE, 8, false) + var resInt8 int8 = int8(res.(int64)) + resPtr = &resInt8 case yang.Yint16: res, err = DbValToInt(dbFldVal, INTBASE, 16, false) + var resInt16 int16 = int16(res.(int64)) + resPtr = &resInt16 case yang.Yint32: res, err = DbValToInt(dbFldVal, INTBASE, 32, false) + var resInt32 int32 = int32(res.(int64)) + resPtr = &resInt32 case yang.Yuint8: res, err = DbValToInt(dbFldVal, INTBASE, 8, true) + var resUint8 uint8 = uint8(res.(uint64)) + resPtr = &resUint8 case yang.Yuint16: res, err = DbValToInt(dbFldVal, INTBASE, 16, true) + var resUint16 uint16 = uint16(res.(uint64)) + resPtr = &resUint16 case yang.Yuint32: res, err = DbValToInt(dbFldVal, INTBASE, 32, true) + var resUint32 uint32 = uint32(res.(uint64)) + resPtr = &resUint32 case yang.Ybool: if res, err = strconv.ParseBool(dbFldVal); err != nil { log.Warningf("Non Bool type for yang leaf-list item %v", dbFldVal) } - case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion,yang.Yleafref: + var resBool bool = res.(bool) + resPtr = &resBool + case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion, yang.Yleafref: // TODO - handle the union type // Make sure to encode as string, expected by util_types.go: ytypes.yangToJSONType - log.Info("Yenum/Ystring/Yunion(having all members as strings) type for yangXpath ", fldXpath) + xfmrLogInfoAll("Yenum/Ystring/Yunion(having all members as strings) type for yangXpath %v", fldXpath) res = dbFldVal + var resString string = res.(string) + resPtr = &resString case yang.Yempty: logStr := fmt.Sprintf("Yang data type for xpath %v is Yempty.", fldXpath) - log.Error(logStr) + log.Warning(logStr) err = errors.New(logStr) default: logStr := fmt.Sprintf("Unrecognized/Unhandled yang-data type(%v) for xpath %v.", fldXpath, yang.TypeKindToName[yngTerminalNdDtType]) - log.Error(logStr) + log.Warning(logStr) err = errors.New(logStr) } - return res, err + return res, resPtr, err } /*convert leaf-list in Db to leaf-list in yang*/ -func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { +func processLfLstDbToYang(fieldXpath string, dbFldVal string, yngTerminalNdDtType yang.TypeKind) []interface{} { valLst := strings.Split(dbFldVal, ",") var resLst []interface{} - const INTBASE = 10 - yngTerminalNdDtType := xDbSpecMap[fieldXpath].dbEntry.Type.Kind + + xfmrLogInfoAll("xpath: %v, dbFldVal: %v", fieldXpath, dbFldVal) switch yngTerminalNdDtType { - case yang.Yenum, yang.Ystring, yang.Yunion, yang.Yleafref: - // TODO handle leaf-ref base type - log.Info("DB leaf-list and Yang leaf-list are of same data-type") + case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion: + // TODO - handle the union type.OC yang should have field xfmr.sonic-yang? + // Make sure to encode as string, expected by util_types.go: ytypes.yangToJSONType: + xfmrLogInfoAll("DB leaf-list and Yang leaf-list are of same data-type") for _, fldVal := range valLst { resLst = append(resLst, fldVal) } default: for _, fldVal := range valLst { - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, fldVal) + resVal, _, err := DbToYangType(yngTerminalNdDtType, fieldXpath, fldVal) if err == nil { resLst = append(resLst, resVal) } @@ -191,16 +212,28 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { return resLst } -func sonicDbToYangTerminalNodeFill(tblName string, field string, value string, resultMap map[string]interface{}) { +func sonicDbToYangTerminalNodeFill(field string, inParamsForGet xlateFromDbParams) { resField := field - if len(value) == 0 { - return + value := "" + + if inParamsForGet.dbDataMap != nil { + tblInstFields, dbDataExists := (*inParamsForGet.dbDataMap)[inParamsForGet.curDb][inParamsForGet.tbl][inParamsForGet.tblKey] + if dbDataExists { + fieldVal, valueExists := tblInstFields.Field[field] + if !valueExists { + return + } + value = fieldVal + } else { + return + } } + if strings.HasSuffix(field, "@") { fldVals := strings.Split(field, "@") resField = fldVals[0] } - fieldXpath := tblName + "/" + resField + fieldXpath := inParamsForGet.tbl + "/" + resField xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] if !ok { log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) @@ -212,46 +245,60 @@ func sonicDbToYangTerminalNodeFill(tblName string, field string, value string, r } yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) + yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind if yangType == YANG_LEAF_LIST { /* this should never happen but just adding for safetty */ if !strings.HasSuffix(field, "@") { log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) return } - resLst := processLfLstDbToYang(fieldXpath, value) - resultMap[resField] = resLst + resLst := processLfLstDbToYang(fieldXpath, value, yngTerminalNdDtType) + inParamsForGet.resultMap[resField] = resLst } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ - yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) + resVal, _, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) if err != nil { log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) } else { - resultMap[resField] = resVal + inParamsForGet.resultMap[resField] = resVal } } - return } -func sonicDbToYangListFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) []typeMapOfInterface { +func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterface { var mapSlice []typeMapOfInterface + dbDataMap := inParamsForGet.dbDataMap + table := inParamsForGet.tbl + dbIdx := inParamsForGet.curDb + xpath := inParamsForGet.xpath dbTblData := (*dbDataMap)[dbIdx][table] - for keyStr, _ := range dbTblData { + for keyStr := range dbTblData { curMap := make(map[string]interface{}) - sonicDbToYangDataFill(uri, xpath, dbIdx, table, keyStr, dbDataMap, curMap) + linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + sonicDbToYangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + inParamsForGet.dbDataMap = dbDataMap dbSpecData, ok := xDbSpecMap[table] if ok && dbSpecData.keyName == nil { yangKeys := yangKeyFromEntryGet(xDbSpecMap[xpath].dbEntry) - sonicKeyDataAdd(dbIdx, yangKeys, keyStr, curMap) + sonicKeyDataAdd(dbIdx, yangKeys, table, keyStr, curMap) } - if curMap != nil { + if len(curMap) > 0 { mapSlice = append(mapSlice, curMap) } } return mapSlice } -func sonicDbToYangDataFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) { +func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { + xpath := inParamsForGet.xpath + uri := inParamsForGet.uri + table := inParamsForGet.tbl + key := inParamsForGet.tblKey + resultMap := inParamsForGet.resultMap + dbDataMap := inParamsForGet.dbDataMap + dbIdx := inParamsForGet.curDb yangNode, ok := xDbSpecMap[xpath] if ok && yangNode.dbEntry != nil { @@ -264,48 +311,86 @@ func sonicDbToYangDataFill(uri string, xpath string, dbIdx db.DBNum, table strin chldYangType := yangTypeGet(xDbSpecMap[chldXpath].dbEntry) if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { - log.Infof("tbl(%v), k(%v), yc(%v)", table, key, yangChldName) + xfmrLogInfoAll("tbl(%v), k(%v), yc(%v)", table, key, yangChldName) fldName := yangChldName if chldYangType == YANG_LEAF_LIST { fldName = fldName + "@" } - sonicDbToYangTerminalNodeFill(table, fldName, (*dbDataMap)[dbIdx][table][key].Field[fldName], resultMap) + curUri := inParamsForGet.uri + "/" + yangChldName + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curUri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + sonicDbToYangTerminalNodeFill(fldName, linParamsForGet) + resultMap = linParamsForGet.resultMap + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CONTAINER { curMap := make(map[string]interface{}) curUri := xpath + "/" + yangChldName // container can have a static key, so extract key for current container _, curKey, curTable := sonicXpathKeyExtract(curUri) // use table-name as xpath from now on - sonicDbToYangDataFill(curUri, curTable, xDbSpecMap[curTable].dbIndex, curTable, curKey, dbDataMap, curMap) + d := inParamsForGet.dbs[xDbSpecMap[curTable].dbIndex] + linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curTable, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + sonicDbToYangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap if len(curMap) > 0 { resultMap[yangChldName] = curMap } else { - log.Infof("Empty container for xpath(%v)", curUri) + xfmrLogInfoAll("Empty container for xpath(%v)", curUri) } + inParamsForGet.dbDataMap = linParamsForGet.dbDataMap + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_LIST { - var mapSlice []typeMapOfInterface - curUri := xpath + "/" + yangChldName - mapSlice = sonicDbToYangListFill(curUri, curUri, dbIdx, table, key, dbDataMap) - if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps - for k, val := range mapSlice[0] { - resultMap[k] = val - } - - } else if len(mapSlice) > 0 { - resultMap[yangChldName] = mapSlice + pathList := strings.Split(uri, "/") + // Skip the list entries if the uri has specific list query + if len(pathList) > SONIC_TABLE_INDEX+1 && !strings.Contains(uri,yangChldName) { + xfmrLogInfoAll("Skipping yangChldName: %v, pathList:%v, len:%v", yangChldName, pathList, len(pathList)) } else { - log.Infof("Empty list for xpath(%v)", curUri) + var mapSlice []typeMapOfInterface + curUri := xpath + "/" + yangChldName + inParamsForGet.uri = curUri + inParamsForGet.xpath = curUri + mapSlice = sonicDbToYangListFill(inParamsForGet) + dbDataMap = inParamsForGet.dbDataMap + if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps + for k, val := range mapSlice[0] { + resultMap[k] = val + } + + } else if len(mapSlice) > 0 { + resultMap[yangChldName] = mapSlice + } else { + xfmrLogInfoAll("Empty list for xpath(%v)", curUri) + } + inParamsForGet.resultMap = resultMap } + } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { + curUri := table + "/" + yangChldName + inParamsForGet.uri = curUri + inParamsForGet.xpath = curUri + inParamsForGet.curDb = xDbSpecMap[table].dbIndex + sonicDbToYangDataFill(inParamsForGet) + dbDataMap = inParamsForGet.dbDataMap + resultMap = inParamsForGet.resultMap + } else { + xfmrLogInfoAll("Not handled case %v", chldXpath) } + } else { + xfmrLogInfoAll("Yang entry not found for %v", chldXpath) } } } - return } /* Traverse db map and create json for cvl yang */ -func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) (string, error) { +func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { + var err error + uri := inParamsForGet.uri + dbDataMap := inParamsForGet.dbDataMap + resultMap := inParamsForGet.resultMap xpath, key, table := sonicXpathKeyExtract(uri) + inParamsForGet.xpath = xpath + inParamsForGet.tbl = table + inParamsForGet.tblKey = key if len(xpath) > 0 { var dbNode *dbInfo @@ -319,12 +404,13 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map if ok && (xDbSpecMap[dbSpecField].fieldType == YANG_LEAF || xDbSpecMap[dbSpecField].fieldType == YANG_LEAF_LIST) { dbNode = xDbSpecMap[dbSpecField] xpath = dbSpecField + inParamsForGet.xpath = xpath } else { dbNode = xDbSpecMap[table] } } } else { - dbNode, _ = xDbSpecMap[xpath] + dbNode = xDbSpecMap[xpath] } if dbNode != nil && dbNode.dbEntry != nil { @@ -333,20 +419,25 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map if len(table) > 0 { cdb = xDbSpecMap[table].dbIndex } + inParamsForGet.curDb = cdb if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { fldName := xDbSpecMap[xpath].dbEntry.Name if yangType == YANG_LEAF_LIST { fldName = fldName + "@" } - sonicDbToYangTerminalNodeFill(table, fldName, (*dbDataMap)[cdb][table][key].Field[fldName], resultMap) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + sonicDbToYangTerminalNodeFill(fldName, linParamsForGet) + resultMap = linParamsForGet.resultMap } else if yangType == YANG_CONTAINER { if len(table) > 0 { xpath = table + inParamsForGet.xpath = xpath } - sonicDbToYangDataFill(uri, xpath, cdb, table, key, dbDataMap, resultMap) + sonicDbToYangDataFill(inParamsForGet) + resultMap = inParamsForGet.resultMap } else if yangType == YANG_LIST { - mapSlice := sonicDbToYangListFill(uri, xpath, cdb, table, key, dbDataMap) + mapSlice := sonicDbToYangListFill(inParamsForGet) if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps for k, val := range mapSlice[0] { resultMap[k] = val @@ -362,9 +453,12 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map } jsonMapData, _ := json.Marshal(resultMap) + isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) jsonData := fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, nil + if isEmptyPayload { + log.Warning("No data available") + } + return jsonData, isEmptyPayload, err } func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (string, string, error) { @@ -372,26 +466,35 @@ func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (stri tableKey := "" for tn, tblData := range dbDataMap { tableName = tn - for kname, _ := range tblData { + for kname := range tblData { tableKey = kname } } return tableName, tableKey, nil } -func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB) (map[db.DBNum]map[string]map[string]db.Value, error) { +func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) (map[db.DBNum]map[string]map[string]db.Value, error) { var err error - dbresult := make(map[db.DBNum]map[string]map[string]db.Value) + dbresult := make(RedisDbMap) dbresult[cdb] = make(map[string]map[string]db.Value) dbFormat := KeySpec{} dbFormat.Ts.Name = tblName - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb if tblKey != "" { + if tblSpecInfo, ok := xDbSpecMap[tblName]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + tblKey, err = dbKeyValueXfmrHandler(CREATE, cdb, tblName, tblKey) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", tblName, tblKey) + return nil, err + } + } + dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) } - err = TraverseDb(dbs, dbFormat, &dbresult, nil) + err = TraverseDb(dbs, dbFormat, &dbresult, nil, dbTblKeyGetCache) if err != nil { - log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + log.Warningf("TraverseDb() didn't fetch data for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) return nil, err } if _, ok := dbresult[cdb]; !ok { @@ -404,126 +507,348 @@ func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string } // Assumption: All tables are from the same DB -func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) error { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - curDbDataMap, err := fillDbDataMapForTbl(inParams.uri, xpath, tbl, inParams.key, inParams.curDb, inParams.dbs) - if err == nil { - mapCopy((*dbDataMap)[inParams.curDb], curDbDataMap[inParams.curDb]) +func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool, xpath string) error { + // skip the query if the table is already visited + if _,ok := (*dbDataMap)[inParams.curDb][tbl]; ok { + if len(inParams.key) > 0 { + if _,ok = (*dbDataMap)[inParams.curDb][tbl][inParams.key]; ok { + return nil + } + } else { + return nil + } } + + terminalNodeGet := false + qdbMapHasTblData := false + qdbMapHasTblKeyData := false + if !xYangSpecMap[xpath].hasNonTerminalNode && len(inParams.key) > 0 { + terminalNodeGet = true + } + if qdbMap, getOk := dbTblKeyGetCache[inParams.curDb]; getOk { + if dbTblData, tblPresent := qdbMap[tbl]; tblPresent { + qdbMapHasTblData = true + if _, keyPresent := dbTblData[inParams.key]; keyPresent { + qdbMapHasTblKeyData = true; + } + } + } + + if !qdbMapHasTblData || (terminalNodeGet && qdbMapHasTblData && !qdbMapHasTblKeyData) { + curDbDataMap, err := fillDbDataMapForTbl(inParams.uri, xpath, tbl, inParams.key, inParams.curDb, inParams.dbs, dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[inParams.curDb], curDbDataMap[inParams.curDb]) + } + } return nil } -func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { +func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error { var tblList []string + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + resultMap := inParamsForGet.resultMap + xpath := inParamsForGet.xpath + tbl := inParamsForGet.tbl + tblKey := inParamsForGet.tblKey + - if tbl == "" && xYangSpecMap[xpath].xfmrTbl != nil { + _, ok := xYangSpecMap[xpath] + if ok { + if xYangSpecMap[xpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[xpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - tblList = xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + tblList, _ = xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) != 0 { for _, curTbl := range tblList { - dbDataFromTblXfmrGet(curTbl, inParams, dbDataMap) + dbDataFromTblXfmrGet(curTbl, inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, xpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } } } + if tbl != "" { + if !contains(tblList, tbl) { + tblList = append(tblList, tbl) + } + } } else if tbl != "" && xYangSpecMap[xpath].xfmrTbl == nil { tblList = append(tblList, tbl) - } else if tbl != "" && xYangSpecMap[xpath].xfmrTbl != nil { - /*key instance level GET, table name and table key filled from xpathKeyExtract which internally calls table transformer*/ - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - dbDataFromTblXfmrGet(tbl, inParams, dbDataMap) - tblList = append(tblList, tbl) + } else if tbl == "" && xYangSpecMap[xpath].xfmrTbl == nil { + // Handling for case: Parent list is not associated with a tableName but has children containers/lists having tableNames. + if tblKey != "" { + var mapSlice []typeMapOfInterface + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + dbDataMap = inParamsForGet.dbDataMap + if err != nil { + log.Infof("Error(%v) returned for %v", err, uri) + } else if ((instMap != nil) && (len(instMap) > 0)) { + mapSlice = append(mapSlice, instMap) + } + if len(mapSlice) > 0 { + listInstanceGet := false + // Check if it is a list instance level Get + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + listInstanceGet = true + for k, v := range mapSlice[0] { + resultMap[k] = v + } + } + if !listInstanceGet { + resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice + } + inParamsForGet.resultMap = resultMap + } + } + } } for _, tbl = range(tblList) { + inParamsForGet.tbl = tbl + tblData, ok := (*dbDataMap)[cdb][tbl] if ok { var mapSlice []typeMapOfInterface - for dbKey, _ := range tblData { - curMap := make(map[string]interface{}) - curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey, dbs[cdb].Opts.KeySeparator) - if len(xYangSpecMap[xpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - mapSlice = append(mapSlice, curMap) - } else { - log.Infof("Empty container returned from overloaded transformer for(\"%v\")", curUri) + for dbKey := range tblData { + inParamsForGet.tblKey = dbKey + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + dbDataMap = inParamsForGet.dbDataMap + if err != nil { + log.Infof("Error(%v) returned for %v", err, uri) + } else if ((instMap != nil) && (len(instMap) > 0)) { + mapSlice = append(mapSlice, instMap) + } + } + + if len(mapSlice) > 0 { + listInstanceGet := false + /*Check if it is a list instance level Get*/ + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + listInstanceGet = true + for k, v := range mapSlice[0] { + resultMap[k] = v } - } else { - _, keyFromCurUri, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri) - if dbKey == keyFromCurUri { - for k, kv := range curKeyMap { - curMap[k] = kv + } + if !listInstanceGet { + if _, specOk := xYangSpecMap[xpath]; specOk { + if _, ok := resultMap[xYangSpecMap[xpath].yangEntry.Name]; ok { + mlen := len(resultMap[xYangSpecMap[xpath].yangEntry.Name].([]typeMapOfInterface)) + for i := 0; i < mlen; i++ { + mapSlice = append(mapSlice, resultMap[xYangSpecMap[xpath].yangEntry.Name].([]typeMapOfInterface)[i]) } - curXpath, _ := XfmrRemoveXPATHPredicates(curUri) - yangDataFill(dbs, ygRoot, curUri, curXpath, dbDataMap, curMap, tbl, dbKey, cdb, validate) - mapSlice = append(mapSlice, curMap) + } + resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice + inParamsForGet.resultMap = resultMap } } - } - if len(mapSlice) > 0 { - resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice } else { - log.Infof("Empty slice for (\"%v\").\r\n", uri) + xfmrLogInfoAll("Empty slice for (\"%v\").\r\n", uri) } } }// end of tblList for + return nil } -func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, tbl string, tblKey string) (map[string]interface{}, error) { - log.Infof("Received xpath - %v, uri - %v, table - %v, table key - %v", xpath, uri, tbl, tblKey) +func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) (typeMapOfInterface,error) { + + var err error + curMap := make(map[string]interface{}) + err = nil + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + xpath := inParamsForGet.xpath + tbl := inParamsForGet.tbl + dbKey := inParamsForGet.tblKey + + curKeyMap, curUri, err := dbKeyToYangDataConvert(uri, requestUri, xpath, tbl, dbDataMap, dbKey, dbs[cdb].Opts.KeySeparator, txCache) + if ((err != nil) || (curKeyMap == nil) || (len(curKeyMap) == 0)) { + xfmrLogInfoAll("Skip filling list instance for uri %v since no yang key found corresponding to db-key %v", uri, dbKey) + return curMap, err + } + parentXpath := parentXpathGet(xpath) + _, ok := xYangSpecMap[xpath] + if ok && len(xYangSpecMap[xpath].xfmrFunc) > 0 { + if isFirstCall || (!isFirstCall && (uri != requestUri) && ((len(xYangSpecMap[parentXpath].xfmrFunc) == 0) || + (len(xYangSpecMap[parentXpath].xfmrFunc) > 0 && (xYangSpecMap[parentXpath].xfmrFunc != xYangSpecMap[xpath].xfmrFunc)))) { + xfmrLogInfoAll("Parent subtree already handled cur uri: %v", xpath) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrFunc) + inParamsForGet.ygRoot = ygRoot + inParamsForGet.dbDataMap = dbDataMap + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + } + } + if xYangSpecMap[xpath].hasChildSubTree { + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } else { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + keyFromCurUri := xpathKeyExtRet.dbKey + inParamsForGet.ygRoot = ygRoot + if dbKey == keyFromCurUri || keyFromCurUri == "" { + if dbKey == keyFromCurUri { + for k, kv := range curKeyMap { + curMap[k] = kv + } + } + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } + return curMap, err +} + +func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery bool) (map[string]interface{}, error) { + xfmrLogInfoAll("Received xpath - %v, uri - %v, table - %v, table key - %v", inParamsForGet.xpath, inParamsForGet.uri, inParamsForGet.tbl, inParamsForGet.tblKey) var err error resFldValMap := make(map[string]interface{}) - if xYangSpecMap[xpath].yangEntry == nil { + xpath := inParamsForGet.xpath + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + tbl := inParamsForGet.tbl + tblKey := inParamsForGet.tblKey + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + + _, ok := xYangSpecMap[xpath] + if !ok || xYangSpecMap[xpath].yangEntry == nil { logStr := fmt.Sprintf("No yang entry found for xpath %v.", xpath) err = fmt.Errorf("%v", logStr) return resFldValMap, err } cdb := xYangSpecMap[xpath].dbIndex - if len(xYangSpecMap[xpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - fldValMap, err := leafXfmrHandlerFunc(inParams) + if len(xYangSpecMap[xpath].xfmrField) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + fldValMap, err := leafXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrField) + inParamsForGet.ygRoot = ygRoot + inParamsForGet.dbDataMap = dbDataMap if err != nil { - logStr := fmt.Sprintf("%Failed to get data from overloaded function for %v -v.", uri, err) - err = fmt.Errorf("%v", logStr) + xfmrLogInfoAll("No data from field transformer for %v: %v.", uri, err) return resFldValMap, err } - if fldValMap != nil { - for lf, val := range fldValMap { + if (uri == requestUri) { + yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + if len(fldValMap) == 0 { + // field transformer returns empty map when no data in DB + if ((yangType == YANG_LEAF) || ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))))) { + log.Warningf("Field transformer returned empty data , uri - %v", requestUri) + err = tlerr.NotFoundError{Format:"Resource not found"} + return resFldValMap, err + } + } else { + if ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + return resFldValMap, nil + } + } + } + for lf, val := range fldValMap { resFldValMap[lf] = val - } - } + } } else { dbFldName := xYangSpecMap[xpath].fieldName - if dbFldName == "NONE" { + if dbFldName == XFMR_NONE_STRING { return resFldValMap, err } /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + yngTerminalNdDtType := xYangSpecMap[xpath].yangEntry.Type.Kind if yangType == YANG_LEAF_LIST { dbFldName += "@" val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] + leafLstInstGetReq := false + + if terminalNodeQuery && ((strings.HasSuffix(requestUri, "]")) || (strings.HasSuffix(requestUri, "]/"))) { + xfmrLogInfoAll("Request URI is leaf-list instance GET - %v", requestUri) + leafLstInstGetReq = true + } if ok { - resLst := processLfLstDbToYang(xpath, val) - resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resLst + if leafLstInstGetReq { + leafListInstVal, valErr := extractLeafListInstFromUri(requestUri) + if valErr != nil { + return resFldValMap, valErr + } + dbSpecField := tbl + "/" + strings.TrimSuffix(dbFldName, "@") + dbSpecFieldInfo, dbSpecOk := xDbSpecMap[dbSpecField] + if dbSpecOk && dbSpecFieldInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(CREATE, cdb, tbl, tblKey, dbFldName, leafListInstVal) + retVal, valXfmrErr := valueXfmrHandler(inParams, *dbSpecFieldInfo.xfmrValue) + if valXfmrErr != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", dbSpecField, leafListInstVal, valXfmrErr) + return resFldValMap, valXfmrErr + } + log.Info("valueXfmrHandler() retuned ", retVal) + leafListInstVal = retVal + } + if !leafListInstExists((*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName], leafListInstVal) { + log.Warningf("Queried leaf-list instance does not exists, uri - %v, dbData - %v", requestUri, (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName]) + err = tlerr.NotFoundError{Format:"Resource not found"} + } + if err == nil { + /* Since translib already fills in ygRoot with queried leaf-list instance, do not + fill in resFldValMap or else Unmarshall of payload(resFldValMap) into ygotTgt in + app layer will create duplicate instances in result. + */ + log.Info("Queried leaf-list instance exists but Since translib already fills in ygRoot with queried leaf-list instance do not populate payload.") + } + return resFldValMap, err + } else { + resLst := processLfLstDbToYang(xpath, val, yngTerminalNdDtType) + resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resLst + } + } else { + if leafLstInstGetReq { + log.Warningf("Queried leaf-list does not exist in DB, uri - %v", requestUri) + err = tlerr.NotFoundError{Format:"Resource not found"} + } } } else { val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { - yngTerminalNdDtType := xYangSpecMap[xpath].yangEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, xpath, val) + resVal, _, err := DbToYangType(yngTerminalNdDtType, xpath, val) if err != nil { - log.Error("Failure in converting Db value type to yang type for field", xpath) + log.Warning("Conversion of Db value type to yang type for field didn't happen.", xpath) } else { resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resVal } + } else { + xfmrLogInfoAll("Field value does not exist in DB for - %v" , uri) + err = tlerr.NotFoundError{Format:"Resource not found"} } } } @@ -531,90 +856,199 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string return resFldValMap, err } -func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { +func yangDataFill(inParamsForGet xlateFromDbParams) error { var err error + validate := inParamsForGet.validate isValid := validate + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + resultMap := inParamsForGet.resultMap + xpath := inParamsForGet.xpath + var chldUri string + yangNode, ok := xYangSpecMap[xpath] if ok && yangNode.yangEntry != nil { for yangChldName := range yangNode.yangEntry.Dir { chldXpath := xpath+"/"+yangChldName - chldUri := uri+"/"+yangChldName + if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].nameWithMod != nil { + chldUri = uri+"/"+ *(xYangSpecMap[chldXpath].nameWithMod) + } else { + chldUri = uri+"/"+yangChldName + } + inParamsForGet.xpath = chldXpath + inParamsForGet.uri = chldUri if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].yangEntry != nil { - cdb = xYangSpecMap[chldXpath].dbIndex + cdb := xYangSpecMap[chldXpath].dbIndex + inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.ygRoot = ygRoot // TODO - handle non CONFIG-DB - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) - res := validateHandlerFunc(inParams) - if res != true { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + res := validateHandlerFunc(inParams, xYangSpecMap[chldXpath].validateFunc) + if !res { continue } else { isValid = res } + inParamsForGet.validate = isValid + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } - chldYangType := yangTypeGet(xYangSpecMap[chldXpath].yangEntry) + chldYangType := xYangSpecMap[chldXpath].yangDataType if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { - fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) + if len(xYangSpecMap[xpath].xfmrFunc) > 0 { + continue + } + fldValMap, err := terminalNodeProcess(inParamsForGet, false) + dbDataMap = inParamsForGet.dbDataMap + ygRoot = inParamsForGet.ygRoot if err != nil { - log.Infof("Failed to get data(\"%v\").", chldUri) + xfmrLogInfoAll("Failed to get data(\"%v\").", chldUri) } for lf, val := range fldValMap { resultMap[lf] = val } + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CONTAINER { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + tblKey := xpathKeyExtRet.dbKey + chtbl := xpathKeyExtRet.tableName + inParamsForGet.ygRoot = ygRoot + + if _, ok := (*dbDataMap)[cdb][chtbl]; !ok && len(chtbl) > 0 { + childDBKey := "" + terminalNodeGet := false + qdbMapHasTblData := false + qdbMapHasTblKeyData := false + if !xYangSpecMap[chldXpath].hasNonTerminalNode { + childDBKey = tblKey + terminalNodeGet = true + } + if qdbMap, getOk := inParamsForGet.dbTblKeyGetCache[cdb]; getOk { + if dbTblData, tblPresent := qdbMap[chtbl]; tblPresent { + qdbMapHasTblData = true + if _, keyPresent := dbTblData[tblKey]; keyPresent { + qdbMapHasTblKeyData = true; + } + } + } + + if !qdbMapHasTblData || (terminalNodeGet && qdbMapHasTblData && !qdbMapHasTblKeyData) { + curDbDataMap, err := fillDbDataMapForTbl(chldUri, chldXpath, chtbl, childDBKey, cdb, dbs, inParamsForGet.dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + inParamsForGet.dbDataMap = dbDataMap + } + } + } cname := xYangSpecMap[chldXpath].yangEntry.Name if xYangSpecMap[chldXpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[chldXpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, tblKey, dbDataMap, nil) - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + tblList, _ := xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) > 1 { log.Warningf("Table transformer returned more than one table for container %v", chldXpath) } if len(tblList) == 0 { continue } - dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) - tbl = tblList[0] + dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, chldXpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + chtbl = tblList[0] } } if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) + if (len(xYangSpecMap[xpath].xfmrFunc) == 0) || + (len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + } } - continue + if !xYangSpecMap[chldXpath].hasChildSubTree { + continue + } + } + cmap2 := make(map[string]interface{}) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + err = yangDataFill(linParamsForGet) + cmap2 = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + if err != nil && len(cmap2) == 0 { + xfmrLogInfoAll("Empty container.(\"%v\").\r\n", chldUri) } else { - cmap := make(map[string]interface{}) - err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) + if len(cmap2) > 0 { + resultMap[cname] = cmap2 } + inParamsForGet.resultMap = resultMap } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } else if chldYangType == YANG_LIST { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.ygRoot = ygRoot cdb = xYangSpecMap[chldXpath].dbIndex + inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap = cmap - } else { - log.Infof("Empty list(\"%v\").\r\n", chldUri) + if (len(xYangSpecMap[xpath].xfmrFunc) == 0) || + (len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[chldXpath].xfmrFunc, err) + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } - } else { - ynode, ok := xYangSpecMap[chldXpath] - lTblName := "" - if ok && ynode.tableName != nil { - lTblName = *ynode.tableName + if !xYangSpecMap[chldXpath].hasChildSubTree { + continue } - yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) } + ynode, ok := xYangSpecMap[chldXpath] + lTblName := "" + if ok && ynode.tableName != nil { + lTblName = *ynode.tableName + } + if _, ok := (*dbDataMap)[cdb][lTblName]; !ok && len(lTblName) > 0 { + curDbDataMap, err := fillDbDataMapForTbl(chldUri, chldXpath, lTblName, "", cdb, dbs, inParamsForGet.dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + inParamsForGet.dbDataMap = dbDataMap + } + } + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangListDataFill(linParamsForGet, false) + resultMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.resultMap = resultMap + inParamsForGet.ygRoot = ygRoot + + } else if chldYangType == "choice" || chldYangType == "case" { + yangDataFill(inParamsForGet) + resultMap = inParamsForGet.resultMap + dbDataMap = inParamsForGet.dbDataMap } else { return err } @@ -625,24 +1059,57 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } /* Traverse linear db-map data and add to nested json data */ -func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) (string, error) { +func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { var err error - jsonData := "" + var fldSbtErr error // used only when direct query on leaf/leaf-list having subtree + var fldErr error //used only when direct query on leaf/leaf-list having field transformer + jsonData := "{}" resultMap := make(map[string]interface{}) + d := inParamsForGet.d + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + inParamsForGet.resultMap = resultMap + if isSonicYang(uri) { - return directDbToYangJsonCreate(uri, dbDataMap, resultMap) + return directDbToYangJsonCreate(inParamsForGet) } else { - var d *db.DB - reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) - yangNode, ok := xYangSpecMap[reqXpath] + xpathKeyExtRet, _ := xpathKeyExtract(d, ygRoot, GET, uri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + + inParamsForGet.xpath = xpathKeyExtRet.xpath + inParamsForGet.tbl = xpathKeyExtRet.tableName + inParamsForGet.tblKey = xpathKeyExtRet.dbKey + inParamsForGet.ygRoot = ygRoot + yangNode, ok := xYangSpecMap[xpathKeyExtRet.xpath] if ok { + /* Invoke pre-xfmr is present for the yang module */ + moduleName := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleName) + if modSpecInfo, specOk := xYangSpecMap[moduleName]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre transformer: %v, dbDataMap: %v ", modSpecInfo.xfmrPre, dbDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return jsonData, true, err + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + yangType := yangTypeGet(yangNode.yangEntry) validateHandlerFlag := false tableXfmrFlag := false IsValidate := false - if len(xYangSpecMap[reqXpath].validateFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, keyName, dbDataMap, nil) - res := validateHandlerFunc(inParams) + if len(xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + res := validateHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if !res { validateHandlerFlag = true /* cannot immediately return from here since reXpath yangtype decides the return type */ @@ -650,6 +1117,7 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db IsValidate = res } } + inParamsForGet.validate = IsValidate isList := false switch yangType { case YANG_LIST: @@ -657,89 +1125,147 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db case YANG_LEAF, YANG_LEAF_LIST, YANG_CONTAINER: isList = false default: - log.Infof("Unknown yang object type for path %v", reqXpath) + xfmrLogInfo("Unknown yang object type for path %v", xpathKeyExtRet.xpath) isList = true //do not want non-list processing to happen } /*If yangtype is a list separate code path is to be taken in case of table transformer since that code path already handles the calling of table transformer and subsequent processing */ if (!validateHandlerFlag) && (!isList) { - if xYangSpecMap[reqXpath].xfmrTbl != nil { - xfmrTblFunc := *xYangSpecMap[reqXpath].xfmrTbl + if xYangSpecMap[xpathKeyExtRet.xpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[xpathKeyExtRet.xpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, keyName, dbDataMap, nil) - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + tblList, _ := xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) > 1 { - log.Warningf("Table transformer returned more than one table for container %v", reqXpath) - tableXfmrFlag = true + log.Warningf("Table transformer returned more than one table for container %v", xpathKeyExtRet.xpath) } if len(tblList) == 0 { - log.Warningf("Table transformer returned no table for conatiner %v", reqXpath) + log.Warningf("Table transformer returned no table for conatiner %v", xpathKeyExtRet.xpath) tableXfmrFlag = true } if !tableXfmrFlag { - dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) + for _, tbl := range tblList { + dbDataFromTblXfmrGet(tbl, inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, xpathKeyExtRet.xpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } } else { - log.Warningf("empty table transformer function name for xpath - %v", reqXpath) + log.Warningf("empty table transformer function name for xpath - %v", xpathKeyExtRet.xpath) tableXfmrFlag = true } } } for { + done := true if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { - yangName := xYangSpecMap[reqXpath].yangEntry.Name + yangName := xYangSpecMap[xpathKeyExtRet.xpath].yangEntry.Name if validateHandlerFlag || tableXfmrFlag { resultMap[yangName] = "" break } - tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) - fldValMap, err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) - if err != nil { - log.Infof("Empty terminal node (\"%v\").", uri) + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + fldSbtErr = xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if fldSbtErr != nil { + /*For request Uri pointing to leaf/leaf-list having subtree, error will be propagated + to handle check of leaf/leaf-list-instance existence in Db , which will be performed + by subtree + */ + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + inParamsForGet.ygRoot = ygRoot + break + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } else { + tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) + inParamsForGet.tbl = tbl + inParamsForGet.tblKey = key + var fldValMap map[string]interface{} + fldValMap, fldErr = terminalNodeProcess(inParamsForGet, true) + if ((fldErr != nil) || (len(fldValMap) == 0)) { + if fldErr == nil { + if yangType == YANG_LEAF { + xfmrLogInfo("Empty terminal node (\"%v\").", uri) + fldErr = tlerr.NotFoundError{Format:"Resource Not found"} + } else if ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + jsonMapData, _ := json.Marshal(resultMap) + jsonData = fmt.Sprintf("%v", string(jsonMapData)) + return jsonData, false, nil + } + } + } + resultMap = fldValMap } - resultMap = fldValMap break } else if yangType == YANG_CONTAINER { - cname := xYangSpecMap[reqXpath].yangEntry.Name cmap := make(map[string]interface{}) - resultMap[cname] = cmap + resultMap = cmap if validateHandlerFlag || tableXfmrFlag { break } - if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - cmap, _ = xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap[cname] = cmap + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if err != nil { + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + return jsonData, true, err + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if !xYangSpecMap[xpathKeyExtRet.xpath].hasChildSubTree { + break } - break } - err = yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, IsValidate) + inParamsForGet.resultMap = make(map[string]interface{}) + err = yangDataFill(inParamsForGet) if err != nil { - log.Infof("Empty container(\"%v\").\r\n", uri) + xfmrLogInfo("Empty container(\"%v\").\r\n", uri) } + resultMap = inParamsForGet.resultMap break } else if yangType == YANG_LIST { - if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap = cmap - } else { - log.Infof("Empty list(\"%v\").\r\n", uri) - } - } else { - err = yangListDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, IsValidate) + isFirstCall := true + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { - log.Infof("yangListDataFill failed for list case(\"%v\").\r\n", uri) + if (((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) && (uri == requestUri)) { + // The error handling here is for the deferred resource check error being handled by the subtree for virtual table cases. + log.Warningf("Subtree at list instance level returns error %v for uri - %v", err, uri) + return jsonData, true, err + + } else { + + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + } } + isFirstCall = false + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if !xYangSpecMap[xpathKeyExtRet.xpath].hasChildSubTree { + break + } + } + inParamsForGet.resultMap = make(map[string]interface{}) + err = yangListDataFill(inParamsForGet, isFirstCall) + if err != nil { + xfmrLogInfo("yangListDataFill failed for list case(\"%v\").\r\n", uri) } + resultMap = inParamsForGet.resultMap break } else { - log.Warningf("Unknown yang object type for path %v", reqXpath) + log.Warningf("Unknown yang object type for path %v", xpathKeyExtRet.xpath) + break + } + if done { break } } //end of for @@ -747,20 +1273,21 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db } jsonMapData, _ := json.Marshal(resultMap) + isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) jsonData = fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, nil -} - -func jsonDataPrint(data string) { - fp, err := os.Create("/tmp/dbToYangJson.txt") - if err != nil { - return - } - defer fp.Close() + if fldSbtErr != nil { + /*error should be propagated only when request Uri points to leaf/leaf-list-instance having subtree, + This is to handle check of leaf/leaf-list-instance existence in Db , which will be performed + by subtree, and depending whether queried node exists or not subtree should return error + */ + return jsonData, isEmptyPayload, fldSbtErr + } + if fldErr != nil { + /* error should be propagated only when request Uri points to leaf/leaf-list-instance and the data + is not available(via field-xfmr or field name) + */ + return jsonData, isEmptyPayload, fldErr + } - fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") - fmt.Fprintf (fp, "%v \r\n", data) - fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + return jsonData, isEmptyPayload, nil } - diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index 6bcccb4a627b..eda61b9c88c2 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -19,7 +19,6 @@ package transformer import ( - "errors" "fmt" "github.com/openconfig/ygot/ygot" "os" @@ -27,181 +26,244 @@ import ( "strings" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/openconfig/ygot/ytypes" - "github.com/openconfig/goyang/pkg/yang" - + "github.com/openconfig/goyang/pkg/yang" log "github.com/golang/glog" ) -/* Invoke the post tansformer */ -func postXfmrHandlerFunc(inParams XfmrParams) (map[string]map[string]db.Value, error) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xYangSpecMap[xpath].xfmrPost, inParams) - if err != nil { - return nil, err - } - retData := ret[0].Interface().(map[string]map[string]db.Value) - log.Info("Post Transformer function :", xYangSpecMap[xpath].xfmrPost, " Xpath: ", xpath, " retData: ", retData) - return retData, err -} +var ocbSch, _ = ocbinds.Schema() /* Fill redis-db map with field & value info */ func dataToDBMapAdd(tableName string, dbKey string, result map[string]map[string]db.Value, field string, value string) { - _, ok := result[tableName] - if !ok { - result[tableName] = make(map[string]db.Value) - } + if len(tableName) > 0 { + _, ok := result[tableName] + if !ok { + result[tableName] = make(map[string]db.Value) + } - _, ok = result[tableName][dbKey] - if !ok { - result[tableName][dbKey] = db.Value{Field: make(map[string]string)} - } + if len(dbKey) > 0 { + _, ok = result[tableName][dbKey] + if !ok { + result[tableName][dbKey] = db.Value{Field: make(map[string]string)} + } - if field == "NONE" { - result[tableName][dbKey].Field["NULL"] = "NULL" - return - } + if field == XFMR_NONE_STRING { + if len(result[tableName][dbKey].Field) == 0 { + result[tableName][dbKey].Field["NULL"] = "NULL" + } + return + } - result[tableName][dbKey].Field[field] = value - return + if len(field) > 0 { + result[tableName][dbKey].Field[field] = value + } + } + } } -func tblNameFromTblXfmrGet(xfmrTblFunc string, inParams XfmrParams) (string, error){ - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) +/*use when single table name is expected*/ +func tblNameFromTblXfmrGet(xfmrTblFunc string, inParams XfmrParams, xfmrDbTblKeyCache map[string]tblKeyCache) (string, error){ + var err error + var tblList []string + tblList, err = xfmrTblHandlerFunc(xfmrTblFunc, inParams, xfmrDbTblKeyCache) + if err != nil { + return "", err + } if len(tblList) != 1 { - logStr := fmt.Sprintf("Invalid return value(%v) from table transformer for (%v)", tblList, inParams.uri) - log.Error(logStr) - err := errors.New(logStr) + xfmrLogInfoAll("Uri (\"%v\") translates to 0 or multiple tables instead of single table - %v", inParams.uri, tblList) return "", err } - return tblList[0], nil + return tblList[0], err } /* Fill the redis-db map with data */ -func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey string, result map[string]map[string]db.Value, xpathPrefix string, name string, value interface{}) error { +func mapFillData(xlateParams xlateToParams) error { var dbs [db.MaxDB]*db.DB var err error - xpath := xpathPrefix + "/" + name + xpath := xlateParams.xpath + "/" + xlateParams.name xpathInfo, ok := xYangSpecMap[xpath] - log.Infof("name: \"%v\", xpathPrefix(\"%v\").", name, xpathPrefix) + xfmrLogInfoAll("name: \"%v\", xpathPrefix(\"%v\").", xlateParams.name, xlateParams.xpath) if !ok || xpathInfo == nil { - log.Errorf("Yang path(\"%v\") not found.", xpath) - return errors.New("Invalid URI") + log.Warningf("Yang path(\"%v\") not found.", xpath) + return nil } if xpathInfo.tableName == nil && xpathInfo.xfmrTbl == nil{ - log.Errorf("Table for yang-path(\"%v\") not found.", xpath) - return errors.New("Invalid table name") + log.Warningf("Table for yang-path(\"%v\") not found.", xpath) + return nil } - if len(dbKey) == 0 { - log.Errorf("Table key for yang path(\"%v\") not found.", xpath) - return errors.New("Invalid table key") + if xpathInfo.tableName != nil && *xpathInfo.tableName == XFMR_NONE_STRING { + log.Warningf("Table for yang-path(\"%v\") NONE.", xpath) + return nil } - if xpathInfo.isKey { - return nil + if len(xlateParams.keyName) == 0 { + log.Warningf("Table key for yang path(\"%v\") not found.", xpath) + return nil } tableName := "" if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, "", xlateParams.txCache) // expecting only one table name from tbl-xfmr - tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams) + tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams, xlateParams.xfmrDbTblKeyCache) if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return err + } + if tableName == "" { + log.Warningf("No table name found for uri (\"%v\")", xlateParams.uri) return err } } else { tableName = *xpathInfo.tableName } - if len(xpathInfo.xfmrFunc) > 0 { - uri = uri + "/" + name - - /* field transformer present */ - log.Infof("Transformer function(\"%v\") invoked for yang path(\"%v\").", xpathInfo.xfmrFunc, xpath) - path, _ := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) - for _, p := range path.Elem { - pathSlice := strings.Split(p.Name, ":") - p.Name = pathSlice[len(pathSlice)-1] - if len(p.Key) > 0 { - for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] - } - } - } - ocbSch, _ := ocbinds.Schema() - schRoot := ocbSch.RootSchema() - node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) - log.Info("GetNode data: ", node[0].Data, " nErr :", nErr) - if nErr != nil { - return nErr - } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, node[0].Data) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return err - } - if ret != nil { - retData := ret[0].Interface().(map[string]string) - log.Info("Transformer function :", xpathInfo.xfmrFunc, " Xpath: ", xpath, " retData: ", retData) - for f, v := range retData { - dataToDBMapAdd(tableName, dbKey, result, f, v) - } - } - return nil + // tblXpathMap used for default value processing for a given request + if tblUriMapVal, tblUriMapOk := xlateParams.tblXpathMap[tableName][xlateParams.keyName]; !tblUriMapOk { + if _, tblOk := xlateParams.tblXpathMap[tableName]; !tblOk { + xlateParams.tblXpathMap[tableName] = make(map[string]map[string]bool) + } + tblUriMapVal = map[string]bool{xlateParams.uri: true} + xlateParams.tblXpathMap[tableName][xlateParams.keyName] = tblUriMapVal + } else { + if tblUriMapVal == nil { + tblUriMapVal = map[string]bool{xlateParams.uri: true} + } else { + tblUriMapVal[xlateParams.uri] = true + } + xlateParams.tblXpathMap[tableName][xlateParams.keyName] = tblUriMapVal } - if len(xpathInfo.fieldName) == 0 { - log.Infof("Field for yang-path(\"%v\") not found in DB.", xpath) - return errors.New("Invalid field name") - } - fieldName := xpathInfo.fieldName - valueStr := "" - if xpathInfo.yangEntry.IsLeafList() { - /* Both yang side and Db side('@' suffix field) the data type is leaf-list */ - log.Info("Yang type and Db type is Leaflist for field = ", xpath) - fieldName += "@" - if reflect.ValueOf(value).Kind() != reflect.Slice { - logStr := fmt.Sprintf("Value for yang xpath %v which is a leaf-list should be a slice", xpath) - log.Error(logStr) - err := errors.New(logStr) - return err + curXlateParams := xlateParams + curXlateParams.tableName = tableName + curXlateParams.xpath = xpath + err = mapFillDataUtil(curXlateParams) + return err +} + +func mapFillDataUtil(xlateParams xlateToParams) error { + var dbs [db.MaxDB]*db.DB + + xpathInfo, ok := xYangSpecMap[xlateParams.xpath] + if !ok { + errStr := fmt.Sprintf("Invalid yang-path(\"%v\").", xlateParams.xpath) + return tlerr.InternalError{Format: errStr} } - valData := reflect.ValueOf(value) - for fidx := 0; fidx < valData.Len(); fidx++ { - if fidx > 0 { - valueStr += "," - } - fVal := fmt.Sprintf("%v", valData.Index(fidx).Interface()) - valueStr = valueStr + fVal + + if len(xpathInfo.xfmrField) > 0 { + xlateParams.uri = xlateParams.uri + "/" + xlateParams.name + + /* field transformer present */ + xfmrLogInfoAll("Transformer function(\"%v\") invoked for yang path(\"%v\"). uri: %v", xpathInfo.xfmrField, xlateParams.xpath, xlateParams.uri) + curYgotNodeData, nodeErr := yangNodeForUriGet(xlateParams.uri, xlateParams.ygRoot) + if nodeErr != nil && xlateParams.oper != DELETE { + return nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, xlateParams.keyName, nil, xlateParams.subOpDataMap, curYgotNodeData, xlateParams.txCache) + retData, err := leafXfmrHandler(inParams, xpathInfo.xfmrField) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return err + } + if retData != nil { + xfmrLogInfoAll("Transformer function : %v Xpath : %v retData: %v", xpathInfo.xfmrField, xlateParams.xpath, retData) + for f, v := range retData { + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, f, v) + } + } + return nil + } + + if len(xpathInfo.fieldName) == 0 { + xfmrLogInfoAll("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + errStr := fmt.Sprintf("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + return tlerr.InternalError{Format: errStr} } - log.Infof("leaf-list value after conversion to DB format %v : %v", fieldName, valueStr) + fieldName := xpathInfo.fieldName + valueStr := "" - } else { // xpath is a leaf - valueStr = fmt.Sprintf("%v", value) - if strings.Contains(valueStr, ":") { - valueStr = strings.Split(valueStr, ":")[1] + fieldXpath := xlateParams.tableName + "/" + fieldName + _, ok = xDbSpecMap[fieldXpath] + if !ok { + logStr := fmt.Sprintf("Failed to find the xDbSpecMap: xpath(\"%v\").", fieldXpath) + log.Warning(logStr) + return nil + } + + if xpathInfo.yangEntry.IsLeafList() { + /* Both yang side and Db side('@' suffix field) the data type is leaf-list */ + xfmrLogInfoAll("Yang type and Db type is Leaflist for field = %v", xlateParams.xpath) + fieldName += "@" + if reflect.ValueOf(xlateParams.value).Kind() != reflect.Slice { + logStr := fmt.Sprintf("Value for yang xpath %v which is a leaf-list should be a slice", xlateParams.xpath) + log.Warning(logStr) + return nil } - } + valData := reflect.ValueOf(xlateParams.value) + for fidx := 0; fidx < valData.Len(); fidx++ { + if fidx > 0 { + valueStr += "," + } - dataToDBMapAdd(tableName, dbKey, result, fieldName, valueStr) - log.Infof("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", tableName, dbKey, - fieldName, valueStr) - return nil + // SNC-3626 - string conversion based on the primitive type + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, fieldName, valData.Index(fidx).Interface()) + if err == nil { + if ((strings.Contains(fVal, ":")) && (strings.HasPrefix(fVal, OC_MDL_PFX) || strings.HasPrefix(fVal, IETF_MDL_PFX) || strings.HasPrefix(fVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + fVal = strings.SplitN(fVal, ":", 2)[1] + } + valueStr = valueStr + fVal + } else { + logStr := fmt.Sprintf("Couldn't unmarshal Json to DbData: table(\"%v\") field(\"%v\") value(\"%v\").", xlateParams.tableName, fieldName, valData.Index(fidx).Interface()) + log.Warning(logStr) + return nil + } + } + xfmrLogInfoAll("leaf-list value after conversion to DB format %v : %v", fieldName, valueStr) + + } else { // xpath is a leaf + + // SNC-3626 - string conversion based on the primitive type + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, fieldName, xlateParams.value) + if err == nil { + valueStr = fVal + } else { + logStr := fmt.Sprintf("Couldn't unmarshal Json to DbData: table(\"%v\") field(\"%v\") value(\"%v\").", xlateParams.tableName, fieldName, xlateParams.value) + log.Warning(logStr) + return nil + } + + if ((strings.Contains(valueStr, ":")) && (strings.HasPrefix(valueStr, OC_MDL_PFX) || strings.HasPrefix(valueStr, IETF_MDL_PFX) || strings.HasPrefix(valueStr, IANA_MDL_PFX))) { + // identity-ref/enum might has module prefix + valueStr = strings.SplitN(valueStr, ":", 2)[1] + } + } + + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, fieldName, valueStr) + xfmrLogInfoAll("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", xlateParams.tableName, xlateParams.keyName, + fieldName, valueStr) + return nil } -func sonicYangReqToDbMapCreate(jsonData interface{}, result map[string]map[string]db.Value) error { - if reflect.ValueOf(jsonData).Kind() == reflect.Map { - data := reflect.ValueOf(jsonData) +func sonicYangReqToDbMapCreate(xlateParams xlateToParams) error { + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Map { + data := reflect.ValueOf(xlateParams.jsonData) for _, key := range data.MapKeys() { _, ok := xDbSpecMap[key.String()] if ok { - directDbMapData("", key.String(), data.MapIndex(key).Interface(), result) + directDbMapData("", key.String(), data.MapIndex(key).Interface(), xlateParams.result) } else { - sonicYangReqToDbMapCreate(data.MapIndex(key).Interface(), result) + curXlateParams := xlateParams + curXlateParams.jsonData = data.MapIndex(key).Interface() + sonicYangReqToDbMapCreate(curXlateParams) } } } @@ -213,9 +275,9 @@ func dbMapDataFill(uri string, tableName string, keyName string, d map[string]in for field, value := range d { fieldXpath := tableName + "/" + field if _, fieldOk := xDbSpecMap[fieldXpath]; (fieldOk && (xDbSpecMap[fieldXpath].dbEntry != nil)) { - log.Info("Found non-nil yang entry in xDbSpecMap for field xpath = ", fieldXpath) + xfmrLogInfoAll("Found non-nil yang entry in xDbSpecMap for field xpath = %v", fieldXpath) if xDbSpecMap[fieldXpath].dbEntry.IsLeafList() { - log.Info("Yang type is Leaflist for field = ", field) + xfmrLogInfoAll("Yang type is Leaflist for field = %v", field) field += "@" fieldDt := reflect.ValueOf(value) fieldValue := "" @@ -223,19 +285,27 @@ func dbMapDataFill(uri string, tableName string, keyName string, d map[string]in if fidx > 0 { fieldValue += "," } - fVal := fmt.Sprintf("%v", fieldDt.Index(fidx).Interface()) - fieldValue = fieldValue + fVal + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, field, fieldDt.Index(fidx).Interface()) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + fieldValue = fieldValue + fVal + } } result[tableName][keyName].Field[field] = fieldValue continue } + dbval, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, field, value) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + result[tableName][keyName].Field[field] = dbval + } } else { // should ideally never happen , just adding for safety - log.Info("Did not find entry in xDbSpecMap for field xpath = ", fieldXpath) + xfmrLogInfoAll("Did not find entry in xDbSpecMap for field xpath = %v", fieldXpath) } - result[tableName][keyName].Field[field] = fmt.Sprintf("%v", value) } - return } func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { @@ -248,12 +318,17 @@ func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonDa if i > 0 { keyName += "|" } - keyName += fmt.Sprintf("%v", d[k]) + fieldXpath := tableName + "/" + k + val, err := unmarshalJsonToDbData(dbEntry.Dir[k], fieldXpath, k, d[k]) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + keyName += val + } delete(d, k) } dbMapDataFill(uri, tableName, keyName, d, result) } - return } func directDbMapData(uri string, tableName string, jsonData interface{}, result map[string]map[string]db.Value) bool { @@ -266,7 +341,7 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result if dbSpecData.keyName != nil { key = *dbSpecData.keyName - log.Infof("Fill data for container uri(%v), key(%v)", uri, key) + xfmrLogInfoAll("Fill data for container uri(%v), key(%v)", uri, key) dbMapDataFill(uri, tableName, key, data, result) return true } @@ -278,10 +353,10 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result eType := yangTypeGet(curDbSpecData.dbEntry) switch eType { case "list": - log.Infof("Fill data for list uri(%v)", uri) + xfmrLogInfoAll("Fill data for list uri(%v)", uri) dbMapListDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) default: - log.Infof("Invalid node type for uri(%v)", uri) + xfmrLogInfoAll("Invalid node type for uri(%v)", uri) } } } @@ -289,277 +364,979 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result return true } -/* Get the db table, key and field name for the incoming delete request */ -func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - var err error - if isSonicYang(path) { - xpathPrefix, keyName, tableName := sonicXpathKeyExtract(path) - log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - err = sonicYangReqToDbMapDelete(xpathPrefix, tableName, keyName, result) - } else { - xpathPrefix, keyName, tableName := xpathKeyExtract(d, ygRoot, oper, path) - log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - spec, ok := xYangSpecMap[xpathPrefix] +/* Get the data from incoming update/replace request, create map and fill with dbValue(ie. field:value to write into redis-db */ +func dbMapUpdate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, requestUri string, jsonData interface{}, result map[int]map[db.DBNum]map[string]map[string]db.Value, yangDefValMap map[string]map[string]db.Value, yangAuxValMap map[string]map[string]db.Value, txCache interface{}) error { + xfmrLogInfo("Update/replace req: path(\"%v\").", path) + err := dbMapCreate(d, ygRoot, oper, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) + xfmrLogInfo("Update/replace req: path(\"%v\") result(\"%v\").", path, result) + printDbData(result, nil, "/tmp/yangToDbDataUpRe.txt") + return err +} + +func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) error { + tblData := xlateParams.result[xlateParams.tableName] + var dbs [db.MaxDB]*db.DB + tblName := xlateParams.tableName + dbKey := xlateParams.keyName + for _, tblUri := range tblUriList { + xfmrLogInfoAll("Processing uri %v for default value filling(Table - %v, dbKey - %v)", tblUri, tblName, dbKey) + yangXpath, _, prdErr := XfmrRemoveXPATHPredicates(tblUri) + if prdErr != nil { + continue + } + yangNode, ok := xYangSpecMap[yangXpath] if ok { - if spec.tableName != nil { - result[*spec.tableName] = make(map[string]db.Value) - if len(keyName) > 0 { - result[*spec.tableName][keyName] = db.Value{Field: make(map[string]string)} - if spec.yangEntry != nil && spec.yangEntry.Node.Statement().Keyword == "leaf" { - result[*spec.tableName][keyName].Field[spec.fieldName] = "" + for childName := range yangNode.yangEntry.Dir { + childXpath := yangXpath + "/" + childName + childUri := tblUri + "/" + childName + childNode, ok := xYangSpecMap[childXpath] + if ok { + if (len(childNode.xfmrFunc) > 0) { + xfmrLogInfoAll("Skip default filling since a subtree Xfmr found for path - %v", childXpath) + continue + } + + if (childNode.yangDataType == YANG_LIST || childNode.yangDataType == YANG_CONTAINER) && !childNode.yangEntry.ReadOnly() { + var tblList []string + tblList = append(tblList, childUri) + err := dbMapDefaultFieldValFill(xlateParams, tblList) + if err != nil { + return err + } + } + if (childNode.tableName != nil && *childNode.tableName == tblName) || (childNode.xfmrTbl != nil) { + tblXfmrPresent := false + var inParamsTblXfmr XfmrParams + if childNode.xfmrTbl != nil { + if len(*childNode.xfmrTbl) > 0 { + inParamsTblXfmr = formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, childUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, "", xlateParams.txCache) + //performance optimization - call table transformer only for default val leaf and avoid for other leaves unless its REPLACE operi(aux-map filling) + tblXfmrPresent = true + + } + } + _, ok := tblData[dbKey].Field[childName] + if !ok { + if len(childNode.xfmrField) > 0 { + childYangType := childNode.yangEntry.Type.Kind + var param interface{} + oper := xlateParams.oper + if len(childNode.defVal) > 0 { + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + xfmrLogInfoAll("Update(\"%v\") default: tbl[\"%v\"]key[\"%v\"]fld[\"%v\"] = val(\"%v\").", + childXpath, tblName, dbKey, childNode.fieldName, childNode.defVal) + _, defValPtr, err := DbToYangType(childYangType, childXpath, childNode.defVal) + if err == nil && defValPtr != nil { + param = defValPtr + } else { + xfmrLogInfoAll("Failed to update(\"%v\") default: tbl[\"%v\"]key[\"%v\"]fld[\"%v\"] = val(\"%v\").", + childXpath, tblName, dbKey, childNode.fieldName, childNode.defVal) + } + } else { + if xlateParams.oper != REPLACE { + continue + } + oper = DELETE + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, tblUri+"/"+childName, xlateParams.requestUri, oper, "", nil, xlateParams.subOpDataMap, param, xlateParams.txCache) + retData, err := leafXfmrHandler(inParams, childNode.xfmrField) + if err != nil { + log.Warningf("Default/AuxMap Value filling. Received error %v from %v", err, childNode.xfmrField) + } + if retData != nil { + xfmrLogInfoAll("Transformer function : %v Xpath: %v retData: %v", childNode.xfmrField, childXpath, retData) + for f, v := range retData { + // Fill default value only if value is not available in result Map + // else we overwrite the value filled in resultMap with default value + _, ok := xlateParams.result[tblName][dbKey].Field[f] + if !ok { + if len(childNode.defVal) > 0 { + dataToDBMapAdd(tblName, dbKey, xlateParams.yangDefValMap, f, v) + } else { + // Fill the yangAuxValMap with all fields that are not in either resultMap or defaultValue Map + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, f, "") + } + } + } + } + } else if len(childNode.fieldName) > 0 { + var xfmrErr error + if _, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok { + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + // Fill default value only if value is not available in result Map + // else we overwrite the value filled in resultMap with default value + _, ok = xlateParams.result[tblName][dbKey].Field[childNode.fieldName] + if !ok { + if len(childNode.defVal) > 0 { + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, childXpath, dbKey, xlateParams.jsonData, xlateParams.resultMap, xlateParams.yangDefValMap, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, &xfmrErr, childName, childNode.defVal, tblName) + err := mapFillDataUtil(curXlateParams) + if err != nil { + log.Warningf("Default/AuxMap Value filling. Received error %v from %v", err, childNode.fieldName) + } + } else { + if xlateParams.oper != REPLACE { + continue + } + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, childNode.fieldName, "") + } + } + } + } + } } - } - } else if len(spec.childTable) > 0 { - for _, child := range spec.childTable { - result[child] = make(map[string]db.Value) } } } } - log.Infof("Delete req: path(\"%v\") result(\"%v\").", path, result) - return err + return nil } -func sonicYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName string, result map[string]map[string]db.Value) error { - if (tableName != "") { - // Specific table entry case - result[tableName] = make(map[string]db.Value) - if (keyName != "") { - // Specific key case - var dbVal db.Value - tokens:= strings.Split(xpathPrefix, "/") - if tokens[SONIC_TABLE_INDEX] == tableName { - fieldName := tokens[len(tokens)-1] - dbSpecField := tableName + "/" + fieldName - _, ok := xDbSpecMap[dbSpecField] - if ok { - yangType := xDbSpecMap[dbSpecField].fieldType - // terminal node case - if yangType == YANG_LEAF_LIST { - fieldName = fieldName + "@" - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = "" - } - if yangType == YANG_LEAF { - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = "" - } - } - } - result[tableName][keyName] = dbVal - } else { - // Get all keys - } - } else { - // Get all table entries - // If table name not available in xpath get top container name - _, ok := xDbSpecMap[xpathPrefix] - if ok && xDbSpecMap[xpathPrefix] != nil { - dbInfo := xDbSpecMap[xpathPrefix] - if dbInfo.fieldType == "container" { - for dir, _ := range dbInfo.dbEntry.Dir { - result[dir] = make(map[string]db.Value) - } - } - } - } - return nil +func dbMapDefaultValFill(xlateParams xlateToParams) error { + for tbl, tblData := range xlateParams.result { + for dbKey := range tblData { + var yxpathList []string //contains all uris(with keys) that were traversed for a table while processing the incoming request + if tblUriMapVal, ok := xlateParams.tblXpathMap[tbl][dbKey]; ok { + for tblUri := range tblUriMapVal { + yxpathList = append(yxpathList, tblUri) + } + } + curXlateParams := xlateParams + curXlateParams.tableName = tbl + curXlateParams.keyName = dbKey + if len(yxpathList) > 0 { + err := dbMapDefaultFieldValFill(curXlateParams, yxpathList) + if err != nil { + return err + } + } + } + } + return nil } -/* Get the data from incoming update/replace request, create map and fill with dbValue(ie. field:value to write into redis-db */ -func dbMapUpdate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - log.Infof("Update/replace req: path(\"%v\").", path) - dbMapCreate(d, ygRoot, oper, path, jsonData, result) - log.Infof("Update/replace req: path(\"%v\") result(\"%v\").", path, result) - printDbData(result, "/tmp/yangToDbDataUpRe.txt") - return nil -} /* Get the data from incoming create request, create map and fill with dbValue(ie. field:value to write into redis-db */ -func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - var err error - root := xpathRootNameGet(path) - if isSonicYang(path) { - err = sonicYangReqToDbMapCreate(jsonData, result) +func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, jsonData interface{}, resultMap map[int]RedisDbMap, yangDefValMap map[string]map[string]db.Value, yangAuxValMap map[string]map[string]db.Value, txCache interface{}) error { + var err, xfmrErr error + var cascadeDelTbl []string + var result = make(map[string]map[string]db.Value) + tblXpathMap := make(map[string]map[string]map[string]bool) + subOpDataMap := make(map[int]*RedisDbMap) + root := xpathRootNameGet(uri) + + /* Check if the parent table exists for RFC compliance */ + var exists bool + var dbs [db.MaxDB]*db.DB + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(d, dbs, ygRoot, oper, uri, nil, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + log.Warningf("Cannot perform Operation %v on uri %v due to - %v", oper, uri, err) + return err + } + if !exists { + errStr := fmt.Sprintf("Parent table does not exist for uri(%v)", uri) + return tlerr.NotFoundError{Format: errStr} + } + + xlateToData := formXlateToDbParam(d, ygRoot, oper, root, uri, "", "", jsonData, resultMap, result, txCache, tblXpathMap, subOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", "") + + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) + + if isSonicYang(uri) { + err = sonicYangReqToDbMapCreate(xlateToData) + xpathPrefix, keyName, tableName := sonicXpathKeyExtract(uri) + xfmrLogInfoAll("xpath - %v, keyName - %v, tableName - %v , for uri - %v", xpathPrefix, keyName, tableName, uri) + fldPth := strings.Split(xpathPrefix, "/") + if len(fldPth) > SONIC_FIELD_INDEX { + fldNm := fldPth[SONIC_FIELD_INDEX] + xfmrLogInfoAll("Field Name : %v", fldNm) + if fldNm != "" { + _, ok := xDbSpecMap[tableName] + if ok { + dbSpecField := tableName + "/" + fldNm + _, dbFldok := xDbSpecMap[dbSpecField] + if dbFldok { + /* RFC compliance - REPLACE on leaf/leaf-list becomes UPDATE/merge */ + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + } else { + log.Warningf("For uri - %v, no entry found in xDbSpecMap for table(%v)/field(%v)", uri, tableName, fldNm) + } + } else { + log.Warningf("For uri - %v, no entry found in xDbSpecMap with tableName - %v", uri, tableName) + } + } + } else { + resultMap[oper] = make(RedisDbMap) + resultMap[oper][db.ConfigDB] = result + } } else { - err = yangReqToDbMapCreate(d, ygRoot, oper, root, "", "", jsonData, result) + /* Invoke pre-xfmr is present for the yang module */ + if modSpecInfo, specOk := xYangSpecMap[moduleNm]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + var dbs [db.MaxDB]*db.DB + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", nil, xlateToData.subOpDataMap, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre-transformer: %v, oper: %v, subOpDataMap: %v ", + modSpecInfo.xfmrPre, oper, subOpDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return err + } + } + + err = yangReqToDbMapCreate(xlateToData) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } } if err == nil { - if oper == CREATE { - moduleNm := "/" + strings.Split(path, "/")[1] - log.Infof("Module name for path %s is %s", path, moduleNm) + if !isSonicYang(uri) { + xpath, _, _ := XfmrRemoveXPATHPredicates(uri) + yangNode, ok := xYangSpecMap[xpath] + defSubOpDataMap := make(map[int]*RedisDbMap) + if ok { + xfmrLogInfo("Fill default value for %v, oper(%v)\r\n", uri, oper) + curXlateToParams := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpath, "", jsonData, resultMap, result, txCache, tblXpathMap, defSubOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", "") + curXlateToParams.yangDefValMap = yangDefValMap + curXlateToParams.yangAuxValMap = yangAuxValMap + err = dbMapDefaultValFill(curXlateToParams) + if err != nil { + return err + } + } + + if ok && oper == REPLACE { + if yangNode.yangDataType == YANG_LEAF { + xfmrLogInfo("Change leaf oper to UPDATE for %v, oper(%v)\r\n", uri, oper) + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + result = make(map[string]map[string]db.Value) + } else if yangNode.yangDataType == YANG_LEAF_LIST { + /* RFC compliance - REPLACE on leaf-list becomes UPDATE/merge */ + xfmrLogInfo("Change leaflist oper to UPDATE for %v, oper(%v)\r\n", uri, oper) + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + result = make(map[string]map[string]db.Value) + } + } + + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) if _, ok := xYangSpecMap[moduleNm]; ok { if xYangSpecMap[moduleNm].yangDataType == "container" && len(xYangSpecMap[moduleNm].xfmrPost) > 0 { - log.Info("Invoke post transformer: ", xYangSpecMap[moduleNm].xfmrPost) - dbDataMap := make(map[db.DBNum]map[string]map[string]db.Value) + xfmrLogInfo("Invoke post transformer: %v, result map: %v ", xYangSpecMap[moduleNm].xfmrPost, result) + var dbDataMap = make(RedisDbMap) dbDataMap[db.ConfigDB] = result var dbs [db.MaxDB]*db.DB - inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, path, oper, "", &dbDataMap, nil) - result, err = postXfmrHandlerFunc(inParams) + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", &dbDataMap, subOpDataMap, nil, txCache) + inParams.yangDefValMap = yangDefValMap + result, err = postXfmrHandlerFunc(xYangSpecMap[moduleNm].xfmrPost, inParams) + if err != nil { + return err + } + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } } } else { - log.Errorf("No Entry exists for module %s in xYangSpecMap. Unable to process post xfmr (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) + log.Warningf("No Entry exists for module %s in xYangSpecMap. Unable to process post xfmr (\"%v\") uri(\"%v\") error (\"%v\").", oper, uri, err) } + if len(result) > 0 || len(subOpDataMap) > 0 { + resultMap[oper] = make(RedisDbMap) + resultMap[oper][db.ConfigDB] = result + for op, redisMapPtr := range subOpDataMap { + if redisMapPtr != nil { + if _,ok := resultMap[op]; !ok { + resultMap[op] = make(RedisDbMap) + } + for dbNum, dbMap := range *redisMapPtr { + if _,ok := resultMap[op][dbNum]; !ok { + resultMap[op][dbNum] = make(map[string]map[string]db.Value) + } + mapCopy(resultMap[op][dbNum],dbMap) + } + } + } + } + } + + err = dbDataXfmrHandler(resultMap) + if err != nil { + log.Warningf("Failed in dbdata-xfmr for %v", resultMap) + return err } - printDbData(result, "/tmp/yangToDbDataCreate.txt") + + if (len(cascadeDelTbl) > 0) { + cdErr := handleCascadeDelete(d, resultMap, cascadeDelTbl) + if cdErr != nil { + xfmrLogInfo("Cascade Delete Failed for cascadeDelTbl (%v), Error (%v).", cascadeDelTbl, cdErr) + return cdErr + } + } + + printDbData(resultMap, yangDefValMap, "/tmp/yangToDbDataCreate.txt") } else { - log.Errorf("DBMapCreate req failed for oper (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) + log.Warningf("DBMapCreate req failed for oper (\"%v\") uri(\"%v\") error (\"%v\").", oper, uri, err) } return err } func yangNodeForUriGet(uri string, ygRoot *ygot.GoStruct) (interface{}, error) { - path, _ := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + path, err := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + if path == nil || err != nil { + log.Warningf("For uri %v - StringToPath failure", uri) + errStr := fmt.Sprintf("Ygot stringTopath failed for uri(%v)", uri) + return nil, tlerr.InternalError{Format: errStr} + } + for _, p := range path.Elem { pathSlice := strings.Split(p.Name, ":") p.Name = pathSlice[len(pathSlice)-1] if len(p.Key) > 0 { for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] + // SNC-2126: check the occurrence of ":" + if ((strings.Contains(ent, ":")) && (strings.HasPrefix(ent, OC_MDL_PFX) || strings.HasPrefix(ent, IETF_MDL_PFX) || strings.HasPrefix(ent, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + eslice := strings.SplitN(ent, ":", 2) + // TODO - exclude the prexix by checking enum type + p.Key[ekey] = eslice[len(eslice)-1] + } else { + p.Key[ekey] = ent + } } } } - ocbSch, _ := ocbinds.Schema() schRoot := ocbSch.RootSchema() node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) if nErr != nil { - return nil, nErr + log.Warningf("For uri %v - GetNode failure - %v", uri, nErr) + errStr := fmt.Sprintf("%v", nErr) + return nil, tlerr.InternalError{Format: errStr} + } + if ((node == nil) || (len(node) == 0) || (node[0].Data == nil)) { + log.Warningf("GetNode returned nil for uri %v", uri) + errStr := "GetNode returned nil for the given uri." + return nil, tlerr.InternalError{Format: errStr} } + xfmrLogInfoAll("GetNode data: %v", node[0].Data) return node[0].Data, nil } -func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, xpathPrefix string, keyName string, jsonData interface{}, result map[string]map[string]db.Value) error { - log.Infof("key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - var dbs [db.MaxDB]*db.DB +func yangReqToDbMapCreate(xlateParams xlateToParams) error { + xfmrLogInfoAll("key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + var dbs [db.MaxDB]*db.DB + var retErr error - if reflect.ValueOf(jsonData).Kind() == reflect.Slice { - log.Infof("slice data: key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - jData := reflect.ValueOf(jsonData) - dataMap := make([]interface{}, jData.Len()) - for idx := 0; idx < jData.Len(); idx++ { - dataMap[idx] = jData.Index(idx).Interface() - } - for _, data := range dataMap { - curKey := "" - curUri, _ := uriWithKeyCreate(uri, xpathPrefix, data) - _, ok := xYangSpecMap[xpathPrefix] - if ok && len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { - /* key transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) - if nodeErr != nil { - curYgotNode = nil - } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpathPrefix].xfmrKey), inParams) - if err != nil { - return err - } - if ret != nil { - curKey = ret[0].Interface().(string) - } - } else if xYangSpecMap[xpathPrefix].keyName != nil { - curKey = *xYangSpecMap[xpathPrefix].keyName - } else { - curKey = keyCreate(keyName, xpathPrefix, data, d.Opts.KeySeparator) - } - yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpathPrefix, curKey, data, result) - } - } else { - if reflect.ValueOf(jsonData).Kind() == reflect.Map { - jData := reflect.ValueOf(jsonData) - for _, key := range jData.MapKeys() { - typeOfValue := reflect.TypeOf(jData.MapIndex(key).Interface()).Kind() - - log.Infof("slice/map data: key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - xpath := uri - curUri := uri - curKey := keyName - pathAttr := key.String() - if len(xpathPrefix) > 0 { - if strings.Contains(pathAttr, ":") { - pathAttr = strings.Split(pathAttr, ":")[1] - } - xpath = xpathPrefix + "/" + pathAttr - curUri = uri + "/" + pathAttr - } - _, ok := xYangSpecMap[xpath] - log.Infof("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", - curKey, xpath, curUri) - if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrKey) > 0 { - /* key transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Slice { + xfmrLogInfoAll("slice data: key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + jData := reflect.ValueOf(xlateParams.jsonData) + dataMap := make([]interface{}, jData.Len()) + for idx := 0; idx < jData.Len(); idx++ { + dataMap[idx] = jData.Index(idx).Interface() + } + for _, data := range dataMap { + curKey := "" + curUri, _ := uriWithKeyCreate(xlateParams.uri, xlateParams.xpath, data) + _, ok := xYangSpecMap[xlateParams.xpath] + if ok && len(xYangSpecMap[xlateParams.xpath].xfmrKey) > 0 { + /* key transformer present */ + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + + ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xlateParams.xpath].xfmrKey) + //if key transformer is called without key values in curUri ignore the error + if err != nil && strings.HasSuffix(curUri, "]") { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + curKey = ktRetData + } else if ok && xYangSpecMap[xlateParams.xpath].keyName != nil { + curKey = *xYangSpecMap[xlateParams.xpath].keyName + } else { + curKey = keyCreate(xlateParams.keyName, xlateParams.xpath, data, xlateParams.d.Opts.KeySeparator) + } + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xlateParams.xpath, curKey, data, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "") + retErr = yangReqToDbMapCreate(curXlateParams) + } + } else { + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Map { + jData := reflect.ValueOf(xlateParams.jsonData) + for _, key := range jData.MapKeys() { + typeOfValue := reflect.TypeOf(jData.MapIndex(key).Interface()).Kind() + + xfmrLogInfoAll("slice/map data: key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + xpath := xlateParams.uri + curUri := xlateParams.uri + curKey := xlateParams.keyName + pathAttr := key.String() + if len(xlateParams.xpath) > 0 { + curUri = xlateParams.uri + "/" + pathAttr + if strings.Contains(pathAttr, ":") { + pathAttr = strings.Split(pathAttr, ":")[1] + } + xpath = xlateParams.xpath + "/" + pathAttr + } + _, ok := xYangSpecMap[xpath] + xfmrLogInfoAll("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", + curKey, xpath, curUri) + if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrKey) > 0 { + specYangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) if nodeErr != nil { curYgotNode = nil } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) - if err != nil { - return err - } - if ret != nil { - curKey = ret[0].Interface().(string) + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xpath].xfmrKey) + if ((err != nil) && (specYangType != YANG_LIST || strings.HasSuffix(curUri, "]"))) { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil } - } else if xYangSpecMap[xpath].keyName != nil { + curKey = ktRetData + } else if ok && xYangSpecMap[xpath].keyName != nil { curKey = *xYangSpecMap[xpath].keyName } - if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { - if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { - /* subtree transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) - if nodeErr != nil { - curYgotNode = nil + if ok && (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { + // Call subtree only if start processing for the requestUri. Skip for parent uri traversal + curXpath, _, _ := XfmrRemoveXPATHPredicates(curUri) + reqXpath, _, _ := XfmrRemoveXPATHPredicates(xlateParams.requestUri) + xfmrLogInfoAll("CurUri: %v, requestUri: %v\r\n", curUri, xlateParams.requestUri) + xfmrLogInfoAll("curxpath: %v, requestxpath: %v\r\n", curXpath, reqXpath) + if strings.HasPrefix(curXpath, reqXpath) { + if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xlateParams.xpath] != xYangSpecMap[xpath]) { + /* subtree transformer present */ + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + stRetData, err := xfmrHandler(inParams, xYangSpecMap[xpath].xfmrFunc) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + if stRetData != nil { + mapCopy(xlateParams.result, stRetData) + } + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return nil - } - if ret != nil { - mapCopy(result, ret[0].Interface().(map[string]map[string]db.Value)) + } + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xpath, curKey, jData.MapIndex(key).Interface(), xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "") + retErr = yangReqToDbMapCreate(curXlateParams) + } else { + pathAttr := key.String() + if strings.Contains(pathAttr, ":") { + pathAttr = strings.Split(pathAttr, ":")[1] + } + xpath := xlateParams.xpath + "/" + pathAttr + xfmrLogInfoAll("LEAF Case: xpath: %v, xpathPrefix: %v, pathAttr: %v", xpath, xlateParams.xpath, pathAttr) + /* skip processing for list key-leaf outside of config container(OC yang) directly under the list. + Inside full-spec isKey is set to true for list key-leaf dierctly under the list(outside of config container) + For ietf yang(eg.ietf-ptp) list key-leaf might have a field transformer. + */ + _, ok := xYangSpecMap[xpath] + if ok && ((!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0)) { + if len(xYangSpecMap[xpath].xfmrFunc) == 0 { + value := jData.MapIndex(key).Interface() + xfmrLogInfoAll("data field: key(\"%v\"), value(\"%v\").", key, value) + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, xlateParams.xpath, curKey, xlateParams.jsonData, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, pathAttr, value, "") + retErr = mapFillData(curXlateParams) + if retErr != nil { + log.Warningf("Failed constructing data for db write: key(\"%v\"), value(\"%v\"), path(\"%v\").", + pathAttr, value, xlateParams.xpath) + return retErr + } + } else { + xfmrLogInfoAll("write: key(\"%v\"), xpath(\"%v\"), uri(%v).",key, xpath, xlateParams.uri) + curYgotNode, nodeErr := yangNodeForUriGet(xlateParams.uri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, curKey, nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + stRetData, err := xfmrHandler(inParams, xYangSpecMap[xpath].xfmrFunc) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + if stRetData != nil { + mapCopy(xlateParams.result, stRetData) + } + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } + } + } + } + } + } + } + + return retErr +} + +func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper int, uri string, dbData RedisDbMap) (bool, error) { + var err error + pathList := splitUri(uri) + + xpath, dbKey, table := sonicXpathKeyExtract(uri) + xfmrLogInfoAll("uri: %v xpath: %v table: %v, key: %v", uri, xpath, table, dbKey) + + if (len(table) > 0) && (len(dbKey) > 0) { + tableExists := false + var derr error + if oper == GET { + var cdb db.DBNum = db.ConfigDB + dbInfo, ok := xDbSpecMap[table] + if !ok { + log.Warningf("No entry in xDbSpecMap for xpath %v, for uri - %v", table, uri) + } else { + cdb = dbInfo.dbIndex + } + tableExists = dbTableExistsInDbData(cdb, table, dbKey, dbData) + derr = tlerr.NotFoundError{Format:"Resource not found"} + } else { + // Valid table mapping exists. Read the table entry from DB + tableExists, derr = dbTableExists(d, table, dbKey, oper) + if derr != nil { + return false, derr } - } else { - yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpath, curKey, jData.MapIndex(key).Interface(), result) - } + } + if len(pathList) == SONIC_LIST_INDEX && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) && !tableExists { + // Uri is at /sonic-module:sonic-module/container-table/list + // PATCH opertion permitted only if table exists in DB. + // POST case since the uri is the parent, the parent needs to exist + // PUT case allow operation(Irrespective of table existence update the DB either through CREATE or REPLACE) + // DELETE case Table instance should be available to perform delete else, CVL may throw error + log.Warningf("Parent table %v with key %v does not exist for oper %v in DB", table, dbKey, oper) + err = tlerr.NotFound("Resource not found") + return false, err + } else if len(pathList) > SONIC_LIST_INDEX && !tableExists { + // Uri is at /sonic-module/container-table/list or /sonic-module/container-table/list/leaf + // Parent table should exist for all CRUD cases + log.Warningf("Parent table %v with key %v does not exist in DB", table, dbKey) + err = tlerr.NotFound("Resource not found") + return false, err } else { - pathAttr := key.String() - if strings.Contains(pathAttr, ":") { - pathAttr = strings.Split(pathAttr, ":")[1] - } - value := jData.MapIndex(key).Interface() - log.Infof("data field: key(\"%v\"), value(\"%v\").", key, value) - err := mapFillData(d, ygRoot, oper, uri, curKey, result, xpathPrefix, - pathAttr, value) - if err != nil { - log.Errorf("Failed constructing data for db write: key(\"%v\"), value(\"%v\"), path(\"%v\").", - pathAttr, value, xpathPrefix) - } + // Allow all other operations + return true, err } - } + } else { + // Request is at module level. No need to check for parent table. Hence return true always or + // Request at /sonic-module:sonic-module/container-table level + return true, err } - } - return nil } -/* Debug function to print the map data into file */ -func printDbData(db map[string]map[string]db.Value, fileName string) { - fp, err := os.Create(fileName) - if err != nil { - return - } - defer fp.Close() - - for k, v := range db { - fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") - fmt.Fprintf(fp, "table name : %v\r\n", k) - for ik, iv := range v { - fmt.Fprintf(fp, " key : %v\r\n", ik) - for k, d := range iv.Field { - fmt.Fprintf(fp, " %v :%v\r\n", k, d) - } +/* This function checks the existence of Parent tables in DB for the given URI request + and returns a boolean indicating if the operation is permitted based on the operation type*/ +func verifyParentTable(d *db.DB, dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbData RedisDbMap, txCache interface{}, subOpDataMap map[int]*RedisDbMap) (bool, error) { + xfmrLogInfoAll("Checking for Parent table existence for uri: %v", uri) + if isSonicYang(uri) { + return verifyParentTableSonic(d, dbs, oper, uri, dbData) + } else { + return verifyParentTableOc(d, dbs, ygRoot, oper, uri, dbData, txCache, subOpDataMap) } - } - fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") - return +} + +func verifyParentTblSubtree(dbs [db.MaxDB]*db.DB, uri string, xfmrFuncNm string, oper int, dbData RedisDbMap) (bool, error) { + var inParams XfmrSubscInParams + inParams.uri = uri + inParams.dbDataMap = make(RedisDbMap) + inParams.dbs = dbs + inParams.subscProc = TRANSLATE_SUBSCRIBE + parentTblExists := true + var err error + + st_result, st_err := xfmrSubscSubtreeHandler(inParams, xfmrFuncNm) + if st_result.isVirtualTbl { + xfmrLogInfoAll("Subtree returned Virtual table true.") + goto Exit + } + if st_err != nil { + err = st_err + parentTblExists = false + goto Exit + } else if st_result.dbDataMap != nil && len(st_result.dbDataMap) > 0 { + xfmrLogInfoAll("Subtree subcribe dbData %v", st_result.dbDataMap) + for dbNo, dbMap := range st_result.dbDataMap { + xfmrLogInfoAll("processing Db no - %v", dbNo) + for table, keyInstance := range dbMap { + xfmrLogInfoAll("processing Db table - %v", table) + for dbKey := range keyInstance { + xfmrLogInfoAll("processing Db key - %v", dbKey) + exists := false + if oper != GET { + dptr, derr := db.NewDB(getDBOptions(dbNo)) + if derr != nil { + log.Warningf("Couldn't allocate NewDb/DbOptions for db - %v, while processing uri - %v", dbNo, uri) + err = derr + parentTblExists = false + goto Exit + } + defer dptr.DeleteDB() + exists, err = dbTableExists(dptr, table, dbKey, oper) + } else { + d := dbs[dbNo] + if dbKey == "*" { //dbKey is "*" for GET on entire list + xfmrLogInfoAll("Found table instance in dbData") + goto Exit + } + // GET case - attempt to find in dbData before doing a dbGet in dbTableExists() + exists = dbTableExistsInDbData(dbNo, table, dbKey, dbData) + if exists { + xfmrLogInfoAll("Found table instance in dbData") + goto Exit + } + exists, err = dbTableExists(d, table, dbKey, oper) + } + if !exists || err != nil { + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", table, dbKey, uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + goto Exit + } + } + } + + } + } else { + log.Warningf("No Table information retrieved from subtree for uri %v", uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + goto Exit + } + Exit: + xfmrLogInfoAll("For subtree at uri - %v, returning ,parentTblExists - %v, err - %v", parentTblExists, err) + return parentTblExists, err +} + +func verifyParentTableOc(d *db.DB, dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbData RedisDbMap, txCache interface{}, subOpDataMap map[int]*RedisDbMap) (bool, error) { + var err error + var cdb db.DBNum + uriList := splitUri(uri) + parentTblExists := true + curUri := "/" + yangType := "" + xpath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpathInfo, ok := xYangSpecMap[xpath] + if !ok { + errStr := fmt.Sprintf("No entry found in xYangSpecMap for uri - %v", uri) + err = tlerr.InternalError{Format:errStr} + return false, err + } + yangType = yangTypeGet(xpathInfo.yangEntry) + if yangType == YANG_LEAF_LIST { + /*query is for leaf-list instance, hence remove that from uriList to avoid list-key like processing*/ + if ((strings.HasSuffix(uriList[len(uriList)-1], "]")) || (strings.HasSuffix(uriList[len(uriList)-1], "]/"))) { //splitUri chops off the leaf-list value having square brackets + uriList[len(uriList)-1] = strings.SplitN(uriList[len(uriList)-1], "[", 2)[0] + xfmrLogInfoAll("Uri list after removing leaf-list instance portion - %v", uriList) + } + } + + parentUriList := uriList[:len(uriList)-1] + xfmrLogInfoAll("Parent uri list - %v", parentUriList) + + // Loop for the parent uri to check parent table existence + for idx, path := range parentUriList { + curUri += uriList[idx] + + /* Check for parent table for oc- yang lists*/ + keyList := rgpKeyExtract.FindAllString(path, -1) + if len(keyList) > 0 { + + //Check for subtree existence + curXpath, _, _ := XfmrRemoveXPATHPredicates(curUri) + curXpathInfo, ok := xYangSpecMap[curXpath] + if !ok { + errStr := fmt.Sprintf("No entry found in xYangSpecMap for uri - %v", curUri) + err = tlerr.InternalError{Format:errStr} + parentTblExists = false + break + } + if oper == GET { + cdb = curXpathInfo.dbIndex + xfmrLogInfoAll("db index for curXpath - %v is %v", curXpath, cdb) + d = dbs[cdb] + } + if curXpathInfo.virtualTbl != nil && *curXpathInfo.virtualTbl { + curUri += "/" + continue + } + // Check for subtree case and invoke subscribe xfmr + if len(curXpathInfo.xfmrFunc) > 0 { + xfmrLogInfoAll("Found subtree for uri - %v", curUri) + stParentTblExists := false + stParentTblExists, err = verifyParentTblSubtree(dbs, curUri, curXpathInfo.xfmrFunc, oper, dbData) + if err != nil { + parentTblExists = false + break + } + if !stParentTblExists { + log.Warningf("Parent Table does not exist for uri %v", uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + break + } + } else { + + xfmrLogInfoAll("Check parent table for uri: %v", curUri) + // Get Table and Key only for yang list instances + xpathKeyExtRet, xerr := xpathKeyExtract(d, ygRoot, oper, curUri, uri, nil, subOpDataMap, txCache, nil) + if xerr != nil { + log.Warningf("Failed to get table and key for uri: %v err: %v", curUri, xerr) + err = xerr + parentTblExists = false + break + } + if (xpathKeyExtRet.isVirtualTbl) { + curUri += "/" + continue + } + + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + // Check for Table existence + xfmrLogInfoAll("DB Entry Check for uri: %v table: %v, key: %v", uri, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + existsInDbData := false + if oper == GET { + // GET case - attempt to find in dbData before doing a dbGet in dbTableExists() + existsInDbData = dbTableExistsInDbData(cdb, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, dbData) + } + // Read the table entry from DB + if !existsInDbData { + exists, derr := dbTableExists(d, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + if derr != nil { + return false, derr + } + if !exists { + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, uri) + err = tlerr.NotFound("Resource not found") + break + } + } + } else { + // We always expect a valid table and key to be returned. Else we cannot validate parent check + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, curUri) + err = tlerr.NotFound("Resource not found") + break + } + } + } + curUri += "/" + } + if !parentTblExists { + // For all operations Parent Table has to exist + return false, err + } + + if yangType == YANG_LIST && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) { + // For PATCH request the current table instance should exist for the operation to go through + // For POST since the target URI is the parent URI, it should exist. + // For DELETE we handle the table verification here to avoid any CVL error thrown for delete on non existent table + xfmrLogInfoAll("Check last parent table for uri: %v", uri) + xpath, _, xpathErr := XfmrRemoveXPATHPredicates(uri) + if xpathErr != nil { + log.Warningf("Xpath conversion didn't happen for Uri - %v, due to - %v", uri, xpathErr) + return false, xpathErr + } + xpathInfo, ok := xYangSpecMap[xpath] + if !ok { + err = fmt.Errorf("xYangSpecMap does not contain xpath - %v", xpath) + return false, err + } + virtualTbl := false + if xpathInfo.virtualTbl != nil { + virtualTbl = *xpathInfo.virtualTbl + } + if virtualTbl { + xfmrLogInfoAll("Virtual table at uri - %v", uri) + return true, nil + } + // Check for subtree case and invoke subscribe xfmr + if len(xpathInfo.xfmrFunc) > 0 { + if !((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) {//uri points to entire list + xfmrLogInfoAll("subtree , whole list case for uri - %v", uri) + return true, nil + } + xfmrLogInfoAll("Found subtree for uri - %v", uri) + parentTblExists, err = verifyParentTblSubtree(dbs, uri, xpathInfo.xfmrFunc, oper, dbData) + if err != nil { + return false, err + } + if !parentTblExists { + log.Warningf("Parent Table does not exist for uri %v", uri) + err = tlerr.NotFound("Resource not found") + return false, err + } + return true, nil + } + + xpathKeyExtRet, xerr := xpathKeyExtract(d, ygRoot, oper, uri, uri, nil, subOpDataMap, txCache, nil) + if xerr != nil { + log.Warningf("xpathKeyExtract failed err: %v, table %v, key %v", xerr, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + return false, xerr + } + if xpathKeyExtRet.isVirtualTbl { + return true, nil + } + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + // Read the table entry from DB + exists := false + var derr error + if oper == GET { + // GET case - find in dbData instead of doing a dbGet in dbTableExists() + cdb = xpathInfo.dbIndex + xfmrLogInfoAll("db index for xpath - %v is %v", xpath, cdb) + exists = dbTableExistsInDbData(cdb, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, dbData) + if !exists { + exists, derr = dbTableExists(dbs[cdb], xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + if derr != nil { + return false, derr + } + if !exists { + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, uri) + err = tlerr.NotFound("Resource not found") + } + } + } else { + exists, derr = dbTableExists(d, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + } + if derr != nil { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, derr) + return false, derr + } + if !exists { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } else { + return true, nil + } + } else if !((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) {//uri points to entire list + return true, nil + } else { + log.Warningf("Unable to get valid table and key err: %v, table %v, key %v", xerr, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + return false, xerr + } + } else if (yangType == YANG_CONTAINER && oper == DELETE && ((xpathInfo.keyName != nil && len(*xpathInfo.keyName) > 0) || len(xpathInfo.xfmrKey) > 0)) { + // If the delete is at container level and the container is mapped to a unique table, then check for table existence to avoid CVL throwing error + parentUri := "" + if len(parentUriList) > 0 { + parentUri = strings.Join(parentUriList, "/") + parentUri = "/" + parentUri + } + // Get table for parent xpath + parentTable, perr := dbTableFromUriGet(d, ygRoot, oper, parentUri, uri, nil, txCache, nil) + // Get table for current xpath + xpathKeyExtRet, cerr := xpathKeyExtract(d, ygRoot, oper, uri, uri, nil, subOpDataMap, txCache, nil) + curKey := xpathKeyExtRet.dbKey + curTable := xpathKeyExtRet.tableName + if len(curTable) > 0 { + if perr == nil && cerr == nil && (curTable != parentTable) && len(curKey) > 0 { + exists, derr := dbTableExists(d, curTable, curKey, oper) + if !exists { + return true, derr + } else { + return true, nil + } + } else { + return true, nil + } + } else { + return true, nil + } + } else { + // PUT at list is allowed to do a create if table does not exist else replace OR + // This is a container or leaf at the end of the URI. Parent check already done and hence all operations are allowed + return true, err + } +} + +/* Debug function to print the map data into file */ +func printDbData(resMap map[int]map[db.DBNum]map[string]map[string]db.Value, yangDefValMap map[string]map[string]db.Value, fileName string) { + fp, err := os.Create(fileName) + if err != nil { + return + } + defer fp.Close() + for oper, dbRes := range resMap { + fmt.Fprintf(fp, "-------------------------- REQ DATA -----------------------------\r\n") + fmt.Fprintf(fp, "Oper Type : %v\r\n", oper) + for d, dbMap := range dbRes { + fmt.Fprintf(fp, "DB num : %v\r\n", d) + for k, v := range dbMap { + fmt.Fprintf(fp, "table name : %v\r\n", k) + for ik, iv := range v { + fmt.Fprintf(fp, " key : %v\r\n", ik) + for k, d := range iv.Field { + fmt.Fprintf(fp, " %v :%v\r\n", k, d) + } + } + } + } + } + fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") + fmt.Fprintf(fp, "-------------------------- YANG DEFAULT DATA --------------------\r\n") + for k, v := range yangDefValMap { + fmt.Fprintf(fp, "table name : %v\r\n", k) + for ik, iv := range v { + fmt.Fprintf(fp, " key : %v\r\n", ik) + for k, d := range iv.Field { + fmt.Fprintf(fp, " %v :%v\r\n", k, d) + } + } + } + fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") } diff --git a/translib/transformer/xlate_utils.go b/translib/transformer/xlate_utils.go index 21604160981c..650ff9d7fd7f 100644 --- a/translib/transformer/xlate_utils.go +++ b/translib/transformer/xlate_utils.go @@ -20,16 +20,30 @@ package transformer import ( "fmt" + "errors" "strings" - "reflect" - "regexp" + "regexp" + "runtime" "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/openconfig/goyang/pkg/yang" "github.com/openconfig/gnmi/proto/gnmi" "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ygot/ytypes" + "github.com/Azure/sonic-mgmt-common/translib/ocbinds" log "github.com/golang/glog" + "sync" ) +func initRegex() { + rgpKeyExtract = regexp.MustCompile(`\[([^\[\]]*)\]`) + rgpIpv6 = regexp.MustCompile(`(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)(%.+)?`) + rgpMac = regexp.MustCompile(`([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$`) + rgpIsMac = regexp.MustCompile(`^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$`) + rgpSncKeyExtract = regexp.MustCompile(`\[([^\[\]]*)\]`) + +} + /* Create db key from data xpath(request) */ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { _, ok := xYangSpecMap[xpath] @@ -39,18 +53,25 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string delim := dbKeySep if len(xYangSpecMap[xpath].delim) > 0 { delim = xYangSpecMap[xpath].delim - log.Infof("key concatenater(\"%v\") found for xpath %v ", delim, xpath) + xfmrLogInfoAll("key concatenater(\"%v\") found for xpath %v ", delim, xpath) } if len(keyPrefix) > 0 { keyPrefix += delim } keyVal := "" for i, k := range (strings.Split(yangEntry.Key, " ")) { if i > 0 { keyVal = keyVal + delim } - val := fmt.Sprint(data.(map[string]interface{})[k]) - if strings.Contains(val, ":") { - val = strings.Split(val, ":")[1] + fieldXpath := xpath + "/" + k + fVal, err := unmarshalJsonToDbData(yangEntry.Dir[k], fieldXpath, k, data.(map[string]interface{})[k]) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } + + if ((strings.Contains(fVal, ":")) && + (strings.HasPrefix(fVal, OC_MDL_PFX) || strings.HasPrefix(fVal, IETF_MDL_PFX) || strings.HasPrefix(fVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + fVal = strings.SplitN(fVal, ":", 2)[1] } - keyVal += val + keyVal += fVal } keyPrefix += string(keyVal) } @@ -60,6 +81,7 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string /* Copy redis-db source to destn map */ func mapCopy(destnMap map[string]map[string]db.Value, srcMap map[string]map[string]db.Value) { + mapCopyMutex.Lock() for table, tableData := range srcMap { _, ok := destnMap[table] if !ok { @@ -75,6 +97,44 @@ func mapCopy(destnMap map[string]map[string]db.Value, srcMap map[string]map[stri } } } + mapCopyMutex.Unlock() +} +var mapMergeMutex = &sync.Mutex{} +/* Merge redis-db source to destn map */ +func mapMerge(destnMap map[string]map[string]db.Value, srcMap map[string]map[string]db.Value, oper int) { + mapMergeMutex.Lock() + for table, tableData := range srcMap { + _, ok := destnMap[table] + if !ok { + destnMap[table] = make(map[string]db.Value) + } + for rule, ruleData := range tableData { + _, ok = destnMap[table][rule] + if !ok { + destnMap[table][rule] = db.Value{Field: make(map[string]string)} + } else { + if (oper == DELETE) { + if ((len(destnMap[table][rule].Field) == 0) && (len(ruleData.Field) > 0)) { + continue; + } + if ((len(destnMap[table][rule].Field) > 0) && (len(ruleData.Field) == 0)) { + destnMap[table][rule] = db.Value{Field: make(map[string]string)}; + } + } + } + for field, value := range ruleData.Field { + dval := destnMap[table][rule] + if dval.IsPopulated() && dval.Has(field) && strings.HasSuffix(field, "@") { + attrList := dval.GetList(field) + attrList = append(attrList, value) + dval.SetList(field, attrList) + } else { + destnMap[table][rule].Field[field] = value + } + } + } + } + mapMergeMutex.Unlock() } func parentXpathGet(xpath string) string { @@ -86,11 +146,17 @@ func parentXpathGet(xpath string) string { return path } -func isYangResType(ytype string) bool { - if ytype == "choose" || ytype == "case" { - return true - } - return false +func parentUriGet(uri string) string { + parentUri := "" + if len(uri) > 0 { + uriList := splitUri(uri) + if len(uriList) > 2 { + parentUriList := uriList[:len(uriList)-1] + parentUri = strings.Join(parentUriList, "/") + parentUri = "/" + parentUri + } + } + return parentUri } func yangTypeGet(entry *yang.Entry) string { @@ -100,7 +166,7 @@ func yangTypeGet(entry *yang.Entry) string { return "" } -func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep string) (map[string]interface{}, string, error) { +func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableName string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, dbKey string, dbKeySep string, txCache interface{}) (map[string]interface{}, string, error) { var err error if len(uri) == 0 && len(xpath) == 0 && len(dbKey) == 0 { err = fmt.Errorf("Insufficient input") @@ -109,7 +175,7 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str if _, ok := xYangSpecMap[xpath]; ok { if xYangSpecMap[xpath].yangEntry == nil { - err = fmt.Errorf("Yang Entry not available for xpath ", xpath) + log.Warningf("Yang Entry not available for xpath %v", xpath) return nil, "", nil } } @@ -125,21 +191,23 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str /* if uri contins key, use it else use xpath */ if strings.Contains(uri, "[") { - uriXpath, _ := XfmrRemoveXPATHPredicates(uri) - if (uriXpath == xpath && (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/"))) { - uriWithKeyCreate = false - } + if strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/") { + uriXpath, _, _ := XfmrRemoveXPATHPredicates(uri) + if uriXpath == xpath { + uriWithKeyCreate = false + } + } uriWithKey = fmt.Sprintf("%v", uri) } if len(xYangSpecMap[xpath].xfmrKey) > 0 { var dbs [db.MaxDB]*db.DB - inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, GET, dbKey, nil, nil) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) + inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + inParams.table = tableName + rmap, err := keyXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrKey) if err != nil { return nil, "", err } - rmap := ret[0].Interface().(map[string]interface{}) if uriWithKeyCreate { for k, v := range rmap { uriWithKey += fmt.Sprintf("[%v=%v]", k, v) @@ -155,12 +223,28 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str rmap := make(map[string]interface{}) if len(keyNameList) > 1 { - log.Infof("No key transformer found for multi element yang key mapping to a single redis key string.") - return rmap, uriWithKey, nil + log.Warningf("No key transformer found for multi element yang key mapping to a single redis key string, for uri %v", uri) + errStr := fmt.Sprintf("Error processing key for list %v", uri) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + keyXpath := xpath + "/" + keyNameList[0] + xyangSpecInfo, ok := xYangSpecMap[keyXpath] + if !ok || xyangSpecInfo == nil { + errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + yngTerminalNdDtType := xyangSpecInfo.yangEntry.Type.Kind + resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[0]) + if err != nil { + err = fmt.Errorf("Failure in converting Db value type to yang type for field %v", keyXpath) + return rmap, uriWithKey, err + } else { + rmap[keyNameList[0]] = resVal } - rmap[keyNameList[0]] = keyDataList[0] if uriWithKeyCreate { - uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], keyDataList[0]) + uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], resVal) } return rmap, uriWithKey, nil @@ -188,13 +272,13 @@ func getYangPathFromUri(uri string) (string, error) { path, err = ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) if err != nil { - log.Errorf("Error in uri to path conversion: %v", err) + log.Warningf("Error in uri to path conversion: %v", err) return "", err } yangPath, yperr := ygot.PathToSchemaPath(path) if yperr != nil { - log.Errorf("Error in Gnmi path to Yang path conversion: %v", yperr) + log.Warningf("Error in Gnmi path to Yang path conversion: %v", yperr) return "", yperr } @@ -203,31 +287,113 @@ func getYangPathFromUri(uri string) (string, error) { func yangKeyFromEntryGet(entry *yang.Entry) []string { var keyList []string - for _, key := range strings.Split(entry.Key, " ") { - keyList = append(keyList, key) - } + keyList = append(keyList, strings.Split(entry.Key, " ")...) return keyList } func isSonicYang(path string) bool { - if strings.HasPrefix(path, "/sonic") { - return true - } - return false + return strings.HasPrefix(path, "/sonic") } -func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, keyStr string, resultMap map[string]interface{}) { - var dbOpts db.Options - dbOpts = getDBOptions(dbIndex) - keySeparator := dbOpts.KeySeparator - keyValList := strings.Split(keyStr, keySeparator) +func hasIpv6AddString(val string) bool { + return rgpIpv6.MatchString(val) +} - if len(keyNameList) != len(keyValList) { - return - } +func hasMacAddString(val string) bool { + return rgpMac.MatchString(val) +} + +func isMacAddString(val string) bool { + return rgpIsMac.MatchString(val) +} + +func getYangTerminalNodeTypeName(xpathPrefix string, keyName string) string { + keyXpath := xpathPrefix + "/" + keyName + xfmrLogInfoAll("getYangTerminalNodeTypeName keyXpath: %v ", keyXpath) + dbInfo, ok := xDbSpecMap[keyXpath] + if ok { + yngTerminalNdTyName := dbInfo.dbEntry.Type.Name + xfmrLogInfoAll("yngTerminalNdTyName: %v", yngTerminalNdTyName) + return yngTerminalNdTyName + } + return "" +} + + +func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, xpathPrefix string, keyStr string, resultMap map[string]interface{}) { + var dbOpts db.Options + var keyValList []string + xfmrLogInfoAll("sonicKeyDataAdd keyNameList:%v, keyStr:%v", keyNameList, keyStr) + + dbOpts = getDBOptions(dbIndex) + keySeparator := dbOpts.KeySeparator + /* num of key separators will be less than number of keys */ + if len(keyNameList) == 1 && keySeparator == ":" { + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[0]) + if yngTerminalNdTyName == "mac-address" && isMacAddString(keyStr) { + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else if (yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address") && hasIpv6AddString(keyStr) { + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else { + keyValList = strings.SplitN(keyStr, keySeparator, -1) + xfmrLogInfoAll("Single key non ipv6/mac address for : separator") + } + } else if (strings.Count(keyStr, keySeparator) == len(keyNameList)-1) { + /* number of keys will match number of key values */ + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else { + /* number of dbKey values more than number of keys */ + if keySeparator == ":" && (hasIpv6AddString(keyStr)|| hasMacAddString(keyStr)) { + xfmrLogInfoAll("Key Str has ipv6/mac address with : separator") + if len(keyNameList) == 2 { + valList := strings.SplitN(keyStr, keySeparator, -1) + /* IPV6 address is first entry */ + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[0]) + if yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address" || yngTerminalNdTyName == "mac-address"{ + keyValList = append(keyValList, strings.Join(valList[:len(valList)-2], keySeparator)) + keyValList = append(keyValList, valList[len(valList)-1]) + } else { + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[1]) + if yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address" || yngTerminalNdTyName == "mac-address" { + keyValList = append(keyValList, valList[0]) + keyValList = append(keyValList, strings.Join(valList[1:], keySeparator)) + } else { + xfmrLogInfoAll("No ipv6 or mac address found in value. Cannot split value ") + } + } + xfmrLogInfoAll("KeyValList has %v", keyValList) + } else { + xfmrLogInfoAll("Number of keys : %v", len(keyNameList)) + } + } else { + keyValList = strings.SplitN(keyStr, keySeparator, -1) + xfmrLogInfoAll("Split all keys KeyValList has %v", keyValList) + } + } + xfmrLogInfoAll("yang keys list - %v, xpathprefix - %v, DB-key string - %v, DB-key list after db key separator split - %v, dbIndex - %v", keyNameList, xpathPrefix, keyStr, keyValList, dbIndex) + + if len(keyNameList) != len(keyValList) { + return + } for i, keyName := range keyNameList { - resultMap[keyName] = keyValList[i] + keyXpath := xpathPrefix + "/" + keyName + dbInfo, ok := xDbSpecMap[keyXpath] + var resVal interface{} + resVal = keyValList[i] + if !ok || dbInfo == nil { + log.Warningf("xDbSpecMap entry not found or is nil for xpath %v, hence data-type conversion cannot happen", keyXpath) + } else { + yngTerminalNdDtType := dbInfo.dbEntry.Type.Kind + var err error + resVal, _, err = DbToYangType(yngTerminalNdDtType, keyXpath, keyValList[i]) + if err != nil { + log.Warningf("Data-type conversion unsuccessfull for xpath %v", keyXpath) + resVal = keyValList[i] + } + } + + resultMap[keyName] = resVal } } @@ -241,13 +407,35 @@ func uriWithKeyCreate (uri string, xpathTmplt string, data interface{}) (string, yangEntry := xYangSpecMap[xpathTmplt].yangEntry if yangEntry != nil { for _, k := range (strings.Split(yangEntry.Key, " ")) { - uri += fmt.Sprintf("[%v=%v]", k, data.(map[string]interface{})[k]) + keyXpath := xpathTmplt + "/" + k + if _, keyXpathEntryOk := xYangSpecMap[keyXpath]; !keyXpathEntryOk { + log.Warningf("No entry found in xYangSpec map for xapth %v", keyXpath) + err = fmt.Errorf("No entry found in xYangSpec map for xapth %v", keyXpath) + break + } + keyYangEntry := xYangSpecMap[keyXpath].yangEntry + if keyYangEntry == nil { + log.Warningf("Yang Entry not available for xpath %v", keyXpath) + err = fmt.Errorf("Yang Entry not available for xpath %v", keyXpath) + break + } + keyVal, keyValErr := unmarshalJsonToDbData(keyYangEntry, keyXpath, k, data.(map[string]interface{})[k]) + if keyValErr != nil { + log.Warningf("unmarshalJsonToDbData() didn't unmarshal for key %v with xpath %v", k, keyXpath) + err = keyValErr + break + } + if ((strings.Contains(keyVal, ":")) && (strings.HasPrefix(keyVal, OC_MDL_PFX) || strings.HasPrefix(keyVal, IETF_MDL_PFX) || strings.HasPrefix(keyVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + keyVal = strings.SplitN(keyVal, ":", 2)[1] + } + uri += fmt.Sprintf("[%v=%v]", k, keyVal) } } else { - err = fmt.Errorf("Yang Entry not available for xpath ", xpathTmplt) + err = fmt.Errorf("Yang Entry not available for xpath %v", xpathTmplt) } } else { - err = fmt.Errorf("No entry in xYangSpecMap for xpath ", xpathTmplt) + err = fmt.Errorf("No entry in xYangSpecMap for xpath %v", xpathTmplt) } return uri, err } @@ -260,16 +448,6 @@ func xpathRootNameGet(path string) string { return "" } -func getDbNum(xpath string ) db.DBNum { - _, ok := xYangSpecMap[xpath] - if ok { - xpathInfo := xYangSpecMap[xpath] - return xpathInfo.dbIndex - } - // Default is ConfigDB - return db.ConfigDB -} - func dbToYangXfmrFunc(funcName string) string { return ("DbToYang_" + funcName) } @@ -278,96 +456,43 @@ func uriModuleNameGet(uri string) (string, error) { var err error result := "" if len(uri) == 0 { - log.Error("Empty uri string supplied") + log.Warning("Empty uri string supplied") err = fmt.Errorf("Empty uri string supplied") return result, err } urislice := strings.Split(uri, ":") if len(urislice) == 1 { - log.Errorf("uri string %s does not have module name", uri) - err = fmt.Errorf("uri string does not have module name: ", uri) + log.Warningf("uri string %s does not have module name", uri) + err = fmt.Errorf("uri string does not have module name: %v", uri) return result, err } moduleNm := strings.Split(urislice[0], "/") result = moduleNm[1] if len(strings.Trim(result, " ")) == 0 { - log.Error("Empty module name") + log.Warning("Empty module name") err = fmt.Errorf("No module name found in uri %s", uri) } - log.Info("module name = ", result) + xfmrLogInfo("module name = %v", result) return result, err } -func recMap(rMap interface{}, name []string, id int, max int) { - if id == max || id < 0 { - return - } - val := name[id] - if reflect.ValueOf(rMap).Kind() == reflect.Map { - data := reflect.ValueOf(rMap) - dMap := data.Interface().(map[string]interface{}) - _, ok := dMap[val] - if ok { - recMap(dMap[val], name, id+1, max) - } else { - dMap[val] = make(map[string]interface{}) - recMap(dMap[val], name, id+1, max) - } - } - return -} - -func mapCreate(xpath string) map[string]interface{} { - retMap := make(map[string]interface{}) - if len(xpath) > 0 { - attrList := strings.Split(xpath, "/") - alLen := len(attrList) - recMap(retMap, attrList, 1, alLen) - } - return retMap -} - -func mapInstGet(name []string, id int, max int, inMap interface{}) map[string]interface{} { - if inMap == nil { - return nil - } - result := reflect.ValueOf(inMap).Interface().(map[string]interface{}) - if id == max { - return result - } - val := name[id] - if reflect.ValueOf(inMap).Kind() == reflect.Map { - data := reflect.ValueOf(inMap) - dMap := data.Interface().(map[string]interface{}) - _, ok := dMap[val] - if ok { - result = mapInstGet(name, id+1, max, dMap[val]) - } else { - return result - } - } - return result -} - -func mapGet(xpath string, inMap map[string]interface{}) map[string]interface{} { - attrList := strings.Split(xpath, "/") - alLen := len(attrList) - recMap(inMap, attrList, 1, alLen) - retMap := mapInstGet(attrList, 1, alLen, inMap) - return retMap -} - -func formXfmrInputRequest(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, oper int, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, param interface{}) XfmrParams { +func formXfmrInputRequest(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, oper int, key string, dbDataMap *RedisDbMap, subOpDataMap map[int]*RedisDbMap, param interface{}, txCache interface{}) XfmrParams { var inParams XfmrParams inParams.d = d inParams.dbs = dbs inParams.curDb = cdb inParams.ygRoot = ygRoot inParams.uri = uri + inParams.requestUri = requestUri inParams.oper = oper inParams.key = key inParams.dbDataMap = dbDataMap + inParams.subOpDataMap = subOpDataMap inParams.param = param // generic param + inParams.txCache = txCache.(*sync.Map) + inParams.skipOrdTblChk = new(bool) + inParams.isVirtualTbl = new(bool) + inParams.pCascadeDelTbl = new([]string) return inParams } @@ -407,10 +532,8 @@ func getDBOptions(dbNo db.DBNum) db.Options { switch dbNo { case db.ApplDB, db.CountersDB: opt = getDBOptionsWithSeparator(dbNo, "", ":", ":") - break - case db.FlexCounterDB, db.AsicDB, db.LogLevelDB, db.ConfigDB, db.StateDB: + case db.FlexCounterDB, db.AsicDB, db.LogLevelDB, db.ConfigDB, db.StateDB, db.ErrorDB, db.UserDB: opt = getDBOptionsWithSeparator(dbNo, "", "|", "|") - break } return opt @@ -425,7 +548,29 @@ func getDBOptionsWithSeparator(dbNo db.DBNum, initIndicator string, tableSeparat }) } +func getXpathFromYangEntry(entry *yang.Entry) string { + xpath := "" + if entry != nil { + xpath = entry.Name + entry = entry.Parent + for { + if entry.Parent != nil { + xpath = entry.Name + "/" + xpath + entry = entry.Parent + } else { + // This is the module entry case + xpath = "/" + entry.Name + ":" + xpath + break + } + } + } + return xpath +} + func stripAugmentedModuleNames(xpath string) string { + if !strings.HasPrefix(xpath, "/") { + xpath = "/" + xpath + } pathList := strings.Split(xpath, "/") pathList = pathList[1:] for i, pvar := range pathList { @@ -438,49 +583,74 @@ func stripAugmentedModuleNames(xpath string) string { return path } -func XfmrRemoveXPATHPredicates(xpath string) (string, error) { - pathList := strings.Split(xpath, "/") - pathList = pathList[1:] - for i, pvar := range pathList { - if strings.Contains(pvar, "[") && strings.Contains(pvar, "]") { - si, ei := strings.Index(pvar, "["), strings.Index(pvar, "]") - // substring contains [] entries - if (si < ei) { - pvar = strings.Split(pvar, "[")[0] - pathList[i] = pvar +func XfmrRemoveXPATHPredicates(uri string) (string, []string, error) { + var uriList []string + var pathList []string + uriList = SplitPath(uri) - } else { - // This substring contained a ] before a [. - return "", fmt.Errorf("Incorrect ordering of [] within substring %s of %s, [ pos: %d, ] pos: %d", pvar, xpath, si, ei) - } - } else if strings.Contains(pvar, "[") || strings.Contains(pvar, "]") { - // This substring contained a mismatched pair of []s. - return "", fmt.Errorf("Mismatched brackets within substring %s of %s", pvar, xpath) - } - if (i > 0) && strings.Contains(pvar, ":") { - pvar = strings.Split(pvar,":")[1] - pathList[i] = pvar - } - } - path := "/" + strings.Join(pathList, "/") - return path,nil + // Strip keys for xpath creation + for _, path := range uriList { + si := strings.Index(path, "[") + if si != -1 { + pathList = append(pathList, path[:si]) + } else { + pathList = append(pathList, path) + } + } + + inPath := strings.Join(pathList, "/") + if !strings.HasPrefix(uri, "..") { + inPath = "/" + inPath + } + + xpath := stripAugmentedModuleNames(inPath) + return xpath, uriList, nil } - /* Extract key vars, create db key and xpath */ - func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (string, string, string) { +func replacePrefixWithModuleName(xpath string) (string) { + //Input xpath is after removing the xpath Predicates + var moduleNm string + if _, ok := xYangSpecMap[xpath]; ok { + moduleNm = xYangSpecMap[xpath].dbEntry.Prefix.Parent.NName() + pathList := strings.Split(xpath, ":") + if len(moduleNm) > 0 && len(pathList) == 2 { + xpath = "/" + moduleNm + ":" + pathList[1] + } + } + return xpath +} + + +/* Extract key vars, create db key and xpath */ +func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, requestUri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subOpDataMap map[int]*RedisDbMap, txCache interface{}, xfmrTblKeyCache map[string]tblKeyCache) (xpathTblKeyExtractRet, error) { + xfmrLogInfoAll("In uri(%v), reqUri(%v), oper(%v)", path, requestUri, oper) + var retData xpathTblKeyExtractRet keyStr := "" - tableName := "" - pfxPath := "" - rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) curPathWithKey := "" cdb := db.ConfigDB var dbs [db.MaxDB]*db.DB + var err error + var isUriForListInstance bool + var pathList []string - pfxPath, _ = XfmrRemoveXPATHPredicates(path) - xpathInfo, ok := xYangSpecMap[pfxPath] + retData.xpath = "" + retData.tableName = "" + retData.dbKey = "" + retData.isVirtualTbl = false + + isUriForListInstance = false + retData.xpath, pathList, _ = XfmrRemoveXPATHPredicates(path) + xpathInfo, ok := xYangSpecMap[retData.xpath] if !ok { - log.Errorf("No entry found in xYangSpecMap for xpath %v.", pfxPath) - return pfxPath, keyStr, tableName + log.Warningf("No entry found in xYangSpecMap for xpath %v.", retData.xpath) + return retData, err + } + // for SUBSCRIBE reuestUri = path + requestUriYangType := yangTypeGet(xpathInfo.yangEntry) + if requestUriYangType == YANG_LIST { + if strings.HasSuffix(path, "]") { //uri is for list instance + isUriForListInstance = true + } } cdb = xpathInfo.dbIndex dbOpts := getDBOptions(cdb) @@ -488,26 +658,67 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { if len(xpathInfo.delim) > 0 { keySeparator = xpathInfo.delim } - - for _, k := range strings.Split(path, "/") { - curPathWithKey += k - yangXpath, _ := XfmrRemoveXPATHPredicates(curPathWithKey) - _, ok := xYangSpecMap[yangXpath] + xpathList := strings.Split(retData.xpath, "/") + xpathList = xpathList[1:] + yangXpath := "" + xfmrLogInfoAll("path elements are : %v", pathList) + for i, k := range pathList { + curPathWithKey += "/" + k + callKeyXfmr := true + yangXpath += "/" + xpathList[i] + xpathInfo, ok := xYangSpecMap[yangXpath] if ok { + yangType := yangTypeGet(xpathInfo.yangEntry) + /* when deleting a specific element from leaf-list query uri is of the form + /prefix-path/leafList-field-name[leafList-field-name=value]. + Here the syntax is like a list-key instance enclosed in square + brackets .So avoid list key instance like processing for such a case + */ + if yangType == YANG_LEAF_LIST { + break + } if strings.Contains(k, "[") { if len(keyStr) > 0 { keyStr += keySeparator } if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + if xfmrTblKeyCache != nil { + if tkCache, _ok := xfmrTblKeyCache[curPathWithKey]; _ok { + if len(tkCache.dbKey) != 0 { + keyStr = tkCache.dbKey + callKeyXfmr = false + } + } + } + if callKeyXfmr { xfmrFuncName := yangToDbXfmrFunc(xYangSpecMap[yangXpath].xfmrKey) - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - ret, err := XlateFuncCall(xfmrFuncName, inParams) - if err != nil { - return "", "", "" + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + ret, err := XlateFuncCall(xfmrFuncName, inParams) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + if ret != nil { + keyStr = ret[0].Interface().(string) + } + } else { + ret, err := keyXfmrHandler(inParams, xYangSpecMap[yangXpath].xfmrKey) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + keyStr = ret } - if ret != nil { - keyStr = ret[0].Interface().(string) + if xfmrTblKeyCache != nil { + if _, _ok := xfmrTblKeyCache[curPathWithKey]; !_ok { + xfmrTblKeyCache[curPathWithKey] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[curPathWithKey] + tkCache.dbKey = keyStr + xfmrTblKeyCache[curPathWithKey] = tkCache } + } } else if xYangSpecMap[yangXpath].keyName != nil { keyStr += *xYangSpecMap[yangXpath].keyName } else { @@ -515,50 +726,124 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { There should be key-transformer, if not then the yang key leaves will be concatenated with respective default DB type key-delimiter */ - for idx, kname := range rgp.FindAllString(k, -1) { + for idx, kname := range rgpKeyExtract.FindAllString(k, -1) { if idx > 0 { keyStr += keySeparator } keyl := strings.TrimRight(strings.TrimLeft(kname, "["), "]") - if strings.Contains(keyl, ":") { - keyl = strings.Split(keyl, ":")[1] - } - keyStr += strings.Split(keyl, "=")[1] + keys := strings.Split(keyl, "=") + keyStr += keys[1] } } - } else if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + } else if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + if xfmrTblKeyCache != nil { + if tkCache, _ok := xfmrTblKeyCache[curPathWithKey]; _ok { + if len(tkCache.dbKey) != 0 { + keyStr = tkCache.dbKey + callKeyXfmr = false + } + } + } + if callKeyXfmr { xfmrFuncName := yangToDbXfmrFunc(xYangSpecMap[yangXpath].xfmrKey) - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - ret, err := XlateFuncCall(xfmrFuncName, inParams) - if err != nil { - return "", "", "" + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + ret, err := XlateFuncCall(xfmrFuncName, inParams) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + if ret != nil { + keyStr = ret[0].Interface().(string) + } + } else { + ret, err := keyXfmrHandler(inParams, xYangSpecMap[yangXpath].xfmrKey) + if ((yangType != YANG_LIST) && (err != nil)) { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + keyStr = ret } - if ret != nil { - keyStr = ret[0].Interface().(string) + if xfmrTblKeyCache != nil { + if _, _ok := xfmrTblKeyCache[curPathWithKey]; !_ok { + xfmrTblKeyCache[curPathWithKey] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[curPathWithKey] + tkCache.dbKey = keyStr + xfmrTblKeyCache[curPathWithKey] = tkCache } + } } else if xYangSpecMap[yangXpath].keyName != nil { keyStr += *xYangSpecMap[yangXpath].keyName } } - curPathWithKey += "/" } curPathWithKey = strings.TrimSuffix(curPathWithKey, "/") + if !strings.HasPrefix(curPathWithKey, "/") { + curPathWithKey = "/" + curPathWithKey + } + retData.dbKey = keyStr tblPtr := xpathInfo.tableName - if tblPtr != nil { + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + retData.tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + inParams.dbDataMap = dbDataMap + } + retData.tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) + if inParams.isVirtualTbl != nil { + retData.isVirtualTbl = *(inParams.isVirtualTbl) + } + if err != nil && oper != GET { + return retData, err + } + } + if ((oper == SUBSCRIBE) && (strings.TrimSpace(keyStr) == "") && (requestUriYangType == YANG_LIST) && (!isUriForListInstance)) { + keyStr="*" + } + retData.dbKey = keyStr + xfmrLogInfoAll("Return uri(%v), xpath(%v), key(%v), tableName(%v), isVirtualTbl:%v", path, retData.xpath, keyStr, retData.tableName, retData.isVirtualTbl) + return retData, err +} + +func dbTableFromUriGet(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, subOpDataMap map[int]*RedisDbMap, txCache interface{}, xfmrTblKeyCache map[string]tblKeyCache) (string, error) { + tableName := "" + var err error + cdb := db.ConfigDB + var dbs [db.MaxDB]*db.DB + + xPath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpathInfo, ok := xYangSpecMap[xPath] + if !ok { + log.Warningf("No entry found in xYangSpecMap for xpath %v.", xPath) + return tableName, err + } + + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { tableName = *tblPtr } else if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, oper, "", nil, nil) - tableName, _ = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams) + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) } - return pfxPath, keyStr, tableName - } + return tableName, err +} - func sonicXpathKeyExtract(path string) (string, string, string) { - xpath, keyStr, tableName := "", "", "" +func sonicXpathKeyExtract(path string) (string, string, string) { + xfmrLogInfoAll("In uri(%v)", path) + xpath, keyStr, tableName, fldNm := "", "", "", "" var err error - xpath, err = XfmrRemoveXPATHPredicates(path) + lpath := path + xpath, _, err = XfmrRemoveXPATHPredicates(path) if err != nil { return xpath, keyStr, tableName } - rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) + if xpath != "" { + fldPth := strings.Split(xpath, "/") + if len(fldPth) > SONIC_FIELD_INDEX { + fldNm = fldPth[SONIC_FIELD_INDEX] + xfmrLogInfoAll("Field Name : %v", fldNm) + } + } pathsubStr := strings.Split(path , "/") if len(pathsubStr) > SONIC_TABLE_INDEX { if strings.Contains(pathsubStr[2], "[") { @@ -569,7 +854,7 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { dbInfo, ok := xDbSpecMap[tableName] cdb := db.ConfigDB if !ok { - log.Infof("No entry in xDbSpecMap for xpath %v in order to fetch DB index.", tableName) + xfmrLogInfoAll("No entry in xDbSpecMap for xpath %v in order to fetch DB index", tableName) return xpath, keyStr, tableName } cdb = dbInfo.dbIndex @@ -577,7 +862,16 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { if dbInfo.keyName != nil { keyStr = *dbInfo.keyName } else { - for i, kname := range rgp.FindAllString(path, -1) { + /* chomp off the field portion to avoid processing specific item delete in leaf-list + eg. /sonic-acl:sonic-acl/ACL_TABLE/ACL_TABLE_LIST[aclname=MyACL2_ACL_IPV4]/ports[ports=Ethernet12] + */ + if fldNm != "" { + pathLst := splitUri(path) + xfmrLogInfoAll("pathList after uri split %v", pathLst) + lpath = "/" + strings.Join(pathLst[:SONIC_FIELD_INDEX-1], "/") + xfmrLogInfoAll("path after removing the field portion %v", lpath) + } + for i, kname := range rgpSncKeyExtract.FindAllString(lpath, -1) { if i > 0 { keyStr += dbOpts.KeySeparator } @@ -586,6 +880,444 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { } } } + xfmrLogInfoAll("Return uri(%v), xpath(%v), key(%v), tableName(%v)", path, xpath, keyStr, tableName) return xpath, keyStr, tableName } +func getYangMdlToSonicMdlList(moduleNm string) []string { + var sncMdlList []string + if len(xDbSpecTblSeqnMap) == 0 { + xfmrLogInfo("xDbSpecTblSeqnMap is empty.") + return sncMdlList + } + if strings.HasPrefix(moduleNm, SONIC_MDL_PFX) { + sncMdlList = append(sncMdlList, moduleNm) + } else { + //can be optimized if there is a way to know sonic modules, a given OC-Yang spans over + for sncMdl := range(xDbSpecTblSeqnMap) { + sncMdlList = append(sncMdlList, sncMdl) + } + } + return sncMdlList +} + +func yangFloatIntToGoType(t yang.TypeKind, v float64) (interface{}, error) { + switch t { + case yang.Yint8: + return int8(v), nil + case yang.Yint16: + return int16(v), nil + case yang.Yint32: + return int32(v), nil + case yang.Yuint8: + return uint8(v), nil + case yang.Yuint16: + return uint16(v), nil + case yang.Yuint32: + return uint32(v), nil + } + return nil, fmt.Errorf("unexpected YANG type %v", t) +} + +func unmarshalJsonToDbData(schema *yang.Entry, fieldXpath string, fieldName string, value interface{}) (string, error) { + var data string + + switch v := value.(type) { + case string: + return fmt.Sprintf("%v", v), nil + } + + ykind := schema.Type.Kind + if ykind == yang.Yleafref { + ykind = getLeafrefRefdYangType(ykind, fieldXpath) + } + + switch ykind { + case yang.Ystring, yang.Ydecimal64, yang.Yint64, yang.Yuint64, + yang.Yenum, yang.Ybool, yang.Ybinary, yang.Yidentityref, yang.Yunion: + data = fmt.Sprintf("%v", value) + + case yang.Yint8, yang.Yint16, yang.Yint32, + yang.Yuint8, yang.Yuint16, yang.Yuint32: + pv, err := yangFloatIntToGoType(ykind, value.(float64)) + if err != nil { + errStr := fmt.Sprintf("error parsing %v for schema %s: %v", value, schema.Name, err) + return "", tlerr.InternalError{Format: errStr} + } + data = fmt.Sprintf("%v", pv) + default: + // TODO - bitset, empty + data = fmt.Sprintf("%v", value) + } + + return data, nil +} + +func copyYangXpathSpecData(dstNode *yangXpathInfo, srcNode *yangXpathInfo) { + if dstNode != nil && srcNode != nil { + *dstNode = *srcNode + } +} + +func isJsonDataEmpty(jsonData string) bool { + return string(jsonData) == "{}" +} + +func getFileNmLineNumStr() string { + _, AbsfileName, lineNum, _ := runtime.Caller(2) + fileNmElems := strings.Split(AbsfileName, "/") + fileNm := fileNmElems[len(fileNmElems)-1] + fNmLnoStr := fmt.Sprintf("[%v:%v]", fileNm, lineNum) + return fNmLnoStr +} + +func xfmrLogInfo(format string, args ...interface{}) { + fNmLnoStr := getFileNmLineNumStr() + log.Infof(fNmLnoStr + format, args...) +} + +func xfmrLogInfoAll(format string, args ...interface{}) { + if log.V(5) { + fNmLnoStr := getFileNmLineNumStr() + log.Infof(fNmLnoStr + format, args...) + } +} + +func formXfmrDbInputRequest(oper int, d db.DBNum, tableName string, key string, field string, value string) XfmrDbParams { + var inParams XfmrDbParams + inParams.oper = oper + inParams.dbNum = d + inParams.tableName = tableName + inParams.key = key + inParams.fieldName = field + inParams.value = value + return inParams +} + +func hasKeyValueXfmr(tblName string) bool { + if specTblInfo, ok := xDbSpecMap[tblName]; ok { + for _, lname := range specTblInfo.listName { + listXpath := tblName + "/" + lname + if specListInfo, ok := xDbSpecMap[listXpath]; ok { + for _, key := range specListInfo.keyList { + keyXpath := tblName + "/" + key + if specKeyInfo, ok := xDbSpecMap[keyXpath]; ok { + if specKeyInfo.xfmrValue != nil { + return true + } + } + } + } + } + } + return false +} + +func dbKeyValueXfmrHandler(oper int, dbNum db.DBNum, tblName string, dbKey string) (string, error) { + var err error + var keyValList []string + + xfmrLogInfoAll("dbKeyValueXfmrHandler: oper(%v), db(%v), tbl(%v), dbKey(%v)", + oper, dbNum, tblName, dbKey) + if specTblInfo, ok := xDbSpecMap[tblName]; ok { + for _, lname := range specTblInfo.listName { + listXpath := tblName + "/" + lname + keyMap := make(map[string]interface{}) + + if specListInfo, ok := xDbSpecMap[listXpath]; ok && len(specListInfo.keyList) > 0 { + sonicKeyDataAdd(dbNum, specListInfo.keyList, tblName, dbKey, keyMap) + + if len(keyMap) == len(specListInfo.keyList) { + for _, kname := range specListInfo.keyList { + keyXpath := tblName + "/" + kname + curKeyVal := fmt.Sprintf("%v", keyMap[kname]) + if kInfo, ok := xDbSpecMap[keyXpath]; ok && xDbSpecMap[keyXpath].xfmrValue != nil { + inParams := formXfmrDbInputRequest(oper, dbNum, tblName, dbKey, kname, curKeyVal) + curKeyVal, err = valueXfmrHandler(inParams, *kInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr: keypath(\"%v\") value (\"%v\"):err(%v).", + keyXpath, curKeyVal, err) + return "", err + } + } + keyValList = append(keyValList, curKeyVal) + } + } + } + } + } + + dbOpts := getDBOptions(dbNum) + retKey := strings.Join(keyValList, dbOpts.KeySeparator) + xfmrLogInfoAll("dbKeyValueXfmrHandler: tbl(%v), dbKey(%v), retKey(%v), keyValList(%v)", + tblName, dbKey, retKey, keyValList) + + return retKey, nil +} + +func dbDataXfmrHandler(resultMap map[int]map[db.DBNum]map[string]map[string]db.Value) error { + xfmrLogInfoAll("Received resultMap(%v)", resultMap) + for oper, dbDataMap := range resultMap { + for dbNum, tblData := range dbDataMap { + for tblName, data := range tblData { + if specTblInfo, ok := xDbSpecMap[tblName]; ok && specTblInfo.hasXfmrFn { + skipKeySet := make(map[string]bool) + for dbKey, fldData := range data { + if _, ok := skipKeySet[dbKey]; !ok { + for fld, val := range fldData.Field { + fldName := fld + if strings.HasSuffix(fld, "@") { + fldName = strings.Split(fld, "@")[0] + } + /* check & invoke value-xfmr */ + fldXpath := tblName + "/" + fldName + if fInfo, ok := xDbSpecMap[fldXpath]; ok && fInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(oper, dbNum, tblName, dbKey, fld, val) + retVal, err := valueXfmrHandler(inParams, *fInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", + fldXpath, val, err) + return err + } + resultMap[oper][dbNum][tblName][dbKey].Field[fld] = retVal + } + } + + /* split tblkey and invoke value-xfmr if present */ + if hasKeyValueXfmr(tblName) { + retKey, err := dbKeyValueXfmrHandler(oper, dbNum, tblName, dbKey) + if err != nil { + return err + } + /* cache processed keys */ + skipKeySet[retKey] = true + if dbKey != retKey { + resultMap[oper][dbNum][tblName][retKey] = resultMap[oper][dbNum][tblName][dbKey] + delete(resultMap[oper][dbNum][tblName], dbKey) + } + } + } + } + } + } + } + } + xfmrLogInfoAll("Transformed resultMap(%v)", resultMap) + return nil +} + +func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper int, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool) xlateFromDbParams { + var inParamsForGet xlateFromDbParams + inParamsForGet.d = d + inParamsForGet.dbs = dbs + inParamsForGet.curDb = cdb + inParamsForGet.ygRoot = ygRoot + inParamsForGet.uri = uri + inParamsForGet.requestUri = requestUri + inParamsForGet.xpath = xpath + inParamsForGet.oper = oper + inParamsForGet.tbl = tbl + inParamsForGet.tblKey = tblKey + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.txCache = txCache + inParamsForGet.resultMap = resultMap + inParamsForGet.validate = validate + + return inParamsForGet +} + +func formXlateToDbParam(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, xpathPrefix string, keyName string, jsonData interface{}, resultMap map[int]RedisDbMap, result map[string]map[string]db.Value, txCache interface{}, tblXpathMap map[string]map[string]map[string]bool, subOpDataMap map[int]*RedisDbMap, pCascadeDelTbl *[]string, xfmrErr *error, name string, value interface{}, tableName string) xlateToParams { + var inParamsForSet xlateToParams + inParamsForSet.d = d + inParamsForSet.ygRoot = ygRoot + inParamsForSet.oper = oper + inParamsForSet.uri = uri + inParamsForSet.requestUri = requestUri + inParamsForSet.xpath = xpathPrefix + inParamsForSet.keyName = keyName + inParamsForSet.jsonData = jsonData + inParamsForSet.resultMap = resultMap + inParamsForSet.result = result + inParamsForSet.txCache = txCache.(*sync.Map) + inParamsForSet.tblXpathMap = tblXpathMap + inParamsForSet.subOpDataMap = subOpDataMap + inParamsForSet.pCascadeDelTbl = pCascadeDelTbl + inParamsForSet.xfmrErr = xfmrErr + inParamsForSet.name = name + inParamsForSet.value = value + inParamsForSet.tableName = tableName + + return inParamsForSet +} + +func xlateUnMarshallUri(ygRoot *ygot.GoStruct, uri string) (*interface{}, error) { + if len(uri) == 0 { + errMsg := errors.New("Error: URI is empty") + log.Warning(errMsg) + return nil, errMsg + } + + path, err := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + if err != nil { + return nil, err + } + + for _, p := range path.Elem { + if strings.Contains(p.Name, ":") { + pathSlice := strings.Split(p.Name, ":") + p.Name = pathSlice[len(pathSlice)-1] + } + } + + deviceObj := (*ygRoot).(*ocbinds.Device) + ygNode, _, errYg := ytypes.GetOrCreateNode(ocbSch.RootSchema(), deviceObj, path) + + if errYg != nil { + log.Warning("Error in creating the target object: ", errYg) + return nil, errYg + } + + return &ygNode, nil +} + +func splitUri(uri string) []string { + pathList := SplitPath(uri) + xfmrLogInfoAll("uri: %v ", uri) + xfmrLogInfoAll("uri path elems: %v", pathList) + return pathList +} + +func dbTableExists(d *db.DB, tableName string, dbKey string, oper int) (bool, error) { + var err error + // Read the table entry from DB + if len(tableName) > 0 { + if hasKeyValueXfmr(tableName) { + if oper == GET { //value tranformer callback decides based on oper type + oper = CREATE + } + retKey, err := dbKeyValueXfmrHandler(oper, d.Opts.DBNo, tableName, dbKey) + if err != nil { + return false, err + } + xfmrLogInfoAll("dbKeyValueXfmrHandler() returned db key %v", retKey) + dbKey = retKey + } + + dbTblSpec := &db.TableSpec{Name: tableName} + + if strings.Contains(dbKey, "*") { + keys, derr := d.GetKeysByPattern(dbTblSpec, dbKey) + if derr != nil { + log.Warningf("Failed to get keys for tbl(%v) dbKey pattern %v error: %v", tableName, dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } + xfmrLogInfoAll("keys for table %v are %v", tableName, keys) + if len(keys) > 0 { + return true, nil + } else { + log.Warningf("dbKey %v does not exist in DB for table %v", dbKey, tableName) + err = tlerr.NotFound("Resource not found") + return false, err + } + } else { + + existingEntry, derr := d.GetEntry(dbTblSpec, db.Key{Comp: []string{dbKey}}) + if derr != nil { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", tableName, dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } + return existingEntry.IsPopulated(), err + } + } else { + log.Warning("Empty table name received") + return false, nil + } +} + +func dbTableExistsInDbData(dbNo db.DBNum, table string, dbKey string, dbData RedisDbMap) bool { + xfmrLogInfoAll("received Db no - %v, table - %v, dbkey - %v", dbNo, table, dbKey) + if _, exists := dbData[dbNo][table][dbKey]; exists { + return true + } else { + return false + } +} + +func leafListInstExists(leafListInDbVal string, checkLeafListInstVal string) bool { + /*function to check if leaf-list DB value contains the given instance*/ + exists := false + xfmrLogInfoAll("received value of leaf-list in DB - %v, Value to be checked if exists in leaf-list - %v", leafListInDbVal, checkLeafListInstVal) + leafListItemLst := strings.Split(leafListInDbVal, ",") + for idx := range(leafListItemLst) { + if leafListItemLst[idx] == checkLeafListInstVal { + exists = true + xfmrLogInfoAll("Leaf-list instance exists") + break + } + } + return exists +} + +func extractLeafListInstFromUri(uri string) (string, error) { + /*function to extract leaf-list instance value coming as part of uri + Handling [ ] in value*/ + xfmrLogInfoAll("received uri - %v", uri) + var leafListInstVal string + yangType := "" + err := fmt.Errorf("Unable to extract leaf-list instance value for uri - %v", uri) + + xpath, _, xerr := XfmrRemoveXPATHPredicates(uri) + if !isSonicYang(uri) { + specInfo, ok := xYangSpecMap[xpath] + if !ok { + return leafListInstVal, xerr + } + yangType = yangTypeGet(specInfo.yangEntry) + if !(yangType == YANG_LEAF_LIST) { + return leafListInstVal, err + } + } else { + tokens:= strings.Split(xpath, "/") + fieldName := "" + tableName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + tableName = tokens[SONIC_TABLE_INDEX] + } + dbSpecField := tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok { + yangType := xDbSpecMap[dbSpecField].fieldType + // terminal node case + if !(yangType == YANG_LEAF_LIST) { + return leafListInstVal, err + } + } + } + + //Check if uri has Leaf-list value + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + xpathList := strings.Split(xpath, "/") + ll_name := xpathList[len(xpathList)-1] + ll_inx := strings.LastIndex(uri,ll_name) + if ll_inx != -1 { + ll_value := uri[ll_inx:] + ll_value = strings.TrimSuffix(ll_value, "]") + valueLst := strings.SplitN(ll_value, "=", 2) + leafListInstVal = valueLst[1] + + if ((strings.Contains(leafListInstVal, ":")) && (strings.HasPrefix(leafListInstVal, OC_MDL_PFX) || strings.HasPrefix(leafListInstVal, IETF_MDL_PFX) || strings.HasPrefix(leafListInstVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + leafListInstVal = strings.SplitN(leafListInstVal, ":", 2)[1] + xfmrLogInfoAll("Leaf-list instance value after removing identityref prefix - %v", leafListInstVal) + } + xfmrLogInfoAll("Leaf-list instance value to be returned - %v", leafListInstVal) + + return leafListInstVal, nil + } + } + return leafListInstVal, err +} + diff --git a/translib/transformer/xlate_xfmr_handler.go b/translib/transformer/xlate_xfmr_handler.go new file mode 100644 index 000000000000..40feaeb520b0 --- /dev/null +++ b/translib/transformer/xlate_xfmr_handler.go @@ -0,0 +1,316 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" + log "github.com/golang/glog" +) + +func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) (error) { + xfmrLogInfoAll("Received inParams %v Subtree function name %v", inParams, xfmrFuncNm) + if inParams.uri != inParams.requestUri { + _, yerr := xlateUnMarshallUri(inParams.ygRoot, inParams.uri) + if yerr != nil { + xfmrLogInfoAll("Failed to generate the ygot Node for uri(\"%v\") err(%v).", inParams.uri, yerr) + } + } + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFuncNm), inParams) + if err != nil { + xfmrLogInfoAll("Failed to retrieve data for xpath(\"%v\") err(%v).", inParams.uri, err) + return err + } + if ((ret != nil) && (len(ret)>0)) { + // db to yang subtree xfmr returns err as the only value in return data list from .Call() + if ret[DBTY_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + } + } + } + return err +} + +func leafXfmrHandlerFunc(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]interface{}, error) { + var err error + var fldValMap map[string]interface{} + + xfmrLogInfoAll("Received inParams %v to invoke Field transformer %v", inParams, xfmrFieldFuncNm) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFieldFuncNm), inParams) + if err != nil { + return fldValMap, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == DBTY_FLD_XFMR_RET_ARGS { + // field xfmr returns err as second value in return data list from .Call() + if ret[DBTY_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFieldFuncNm, err) + } + } + } + + if ret[DBTY_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + fldValMap = ret[DBTY_FLD_XFMR_RET_VAL_INDX].Interface().(map[string]interface{}) + } + } + return fldValMap, err +} + +func keyXfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) (map[string]interface{}, error) { + xfmrLogInfoAll("Received inParams %v key transformer function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFuncNm), inParams) + retVal := make(map[string]interface{}) + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == DBTY_KEY_XFMR_RET_ARGS { + // key xfmr returns err as second value in return data list from .Call() + if ret[DBTY_KEY_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_KEY_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[DBTY_KEY_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[DBTY_KEY_XFMR_RET_VAL_INDX].Interface().(map[string]interface{}) + return retVal, nil + } + } + return retVal, nil +} + +func validateHandlerFunc(inParams XfmrParams, validateFuncNm string) (bool) { + xfmrLogInfoAll("Received inParams %v, validate transformer function name %v", inParams, validateFuncNm) + ret, err := XlateFuncCall(validateFuncNm, inParams) + if err != nil { + return false + } + return ret[0].Interface().(bool) +} + +func xfmrTblHandlerFunc(xfmrTblFunc string, inParams XfmrParams, xfmrTblKeyCache map[string]tblKeyCache) ([]string, error) { + + xfmrLogInfoAll("Received inParams %v, table transformer function name %v", inParams, xfmrTblFunc) + if (inParams.oper == GET && xfmrTblKeyCache != nil) { + if tkCache, _ok := xfmrTblKeyCache[inParams.uri]; _ok { + if len(tkCache.dbTblList) > 0 { + return tkCache.dbTblList, nil + } + } + } + + var retTblLst []string + ret, err := XlateFuncCall(xfmrTblFunc, inParams) + if err != nil { + return retTblLst, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == TBL_XFMR_RET_ARGS { + // table xfmr returns err as second value in return data list from .Call() + if ret[TBL_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[TBL_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrTblFunc, err) + return retTblLst, err + } + } + } + + if ret[TBL_XFMR_RET_VAL_INDX].Interface() != nil { + retTblLst = ret[TBL_XFMR_RET_VAL_INDX].Interface().([]string) + } + } + if (inParams.oper == GET && xfmrTblKeyCache != nil) { + if _, _ok := xfmrTblKeyCache[inParams.uri]; !_ok { + xfmrTblKeyCache[inParams.uri] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[inParams.uri] + tkCache.dbTblList = retTblLst + xfmrTblKeyCache[inParams.uri] = tkCache + } + + return retTblLst, err +} + +func valueXfmrHandler(inParams XfmrDbParams, xfmrValueFuncNm string) (string, error) { + xfmrLogInfoAll("Received inParams %v Value transformer name %v", inParams, xfmrValueFuncNm) + + ret, err := XlateFuncCall(xfmrValueFuncNm, inParams) + if err != nil { + return "", err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_FLD_XFMR_RET_ARGS { + // value xfmr returns err as second value in return data list from .Call() + if ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrValueFuncNm, err) + return "", err + } + } + } + + if ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + retVal := ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface().(string) + return retVal, nil + } + } + + return "", err +} + +func leafXfmrHandler(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]string, error) { + xfmrLogInfoAll("Received inParams %v Field transformer name %v", inParams, xfmrFieldFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFieldFuncNm), inParams) + if err != nil { + return nil, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_FLD_XFMR_RET_ARGS { + // field xfmr returns err as second value in return data list from .Call() + if ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFieldFuncNm, err) + return nil, err + } + } + } + + if ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + fldValMap := ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface().(map[string]string) + return fldValMap, nil + } + } else { + retFldValMap := map[string]string{"NULL":"NULL"} + return retFldValMap, nil + } + + return nil, nil +} + +func xfmrHandler(inParams XfmrParams, xfmrFuncNm string) (map[string]map[string]db.Value, error) { + xfmrLogInfoAll("Received inParams %v Subtree function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFuncNm), inParams) + if err != nil { + return nil, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_SBT_XFMR_RET_ARGS { + // subtree xfmr returns err as second value in return data list from .Call() + if ret[YTDB_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return nil, err + } + } + } + if ret[YTDB_SBT_XFMR_RET_VAL_INDX].Interface() != nil { + retMap := ret[YTDB_SBT_XFMR_RET_VAL_INDX].Interface().(map[string]map[string]db.Value) + return retMap, nil + } + } + return nil, nil +} + +func keyXfmrHandler(inParams XfmrParams, xfmrFuncNm string) (string, error) { + xfmrLogInfoAll("Received inParams %v key transformer function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFuncNm), inParams) + retVal := "" + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_KEY_XFMR_RET_ARGS { + // key xfmr returns err as second value in return data list from .Call() + if ret[YTDB_KEY_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_KEY_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[YTDB_KEY_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[YTDB_KEY_XFMR_RET_VAL_INDX].Interface().(string) + return retVal, nil + } + } + return retVal, nil +} + +/* Invoke the post tansformer */ +func postXfmrHandlerFunc(xfmrPost string, inParams XfmrParams) (map[string]map[string]db.Value, error) { + retData := make(map[string]map[string]db.Value) + xfmrLogInfoAll("Received inParams %v, post transformer function name %v", inParams, xfmrPost) + ret, err := XlateFuncCall(xfmrPost, inParams) + if err != nil { + return nil, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == POST_XFMR_RET_ARGS { + // post xfmr returns err as second value in return data list from .Call() + if ret[POST_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[POST_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrPost, err) + return retData, err + } + } + } + if ret[POST_XFMR_RET_VAL_INDX].Interface() != nil { + retData = ret[POST_XFMR_RET_VAL_INDX].Interface().(map[string]map[string]db.Value) + xfmrLogInfoAll("Post Transformer function : %v retData : %v", xfmrPost, retData) + } + } + return retData, err +} + +/* Invoke the pre tansformer */ +func preXfmrHandlerFunc(xfmrPre string, inParams XfmrParams) error { + xfmrLogInfoAll("Received inParams %v, pre transformer function name %v", inParams, xfmrPre) + ret, err := XlateFuncCall(xfmrPre, inParams) + if err != nil { + log.Warningf("Pre-transformer function(\"%v\") returned error - %v.", xfmrPre, err) + return err + } + if ((ret != nil) && (len(ret)>0)) { + if ret[PRE_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[PRE_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrPre, err) + return err + } + } + } + return err +} + diff --git a/translib/transformer/xspec.go b/translib/transformer/xspec.go index 9a6951fee3a0..ae61ad2977c8 100644 --- a/translib/transformer/xspec.go +++ b/translib/transformer/xspec.go @@ -23,8 +23,9 @@ import ( "os" "strings" log "github.com/golang/glog" + "github.com/Azure/sonic-mgmt-common/cvl" "github.com/Azure/sonic-mgmt-common/translib/db" - + "strconv" "github.com/openconfig/goyang/pkg/yang" ) @@ -40,30 +41,101 @@ type yangXpathInfo struct { delim string fieldName string xfmrFunc string + xfmrField string xfmrPost string validateFunc string + rpcFunc string xfmrKey string keyName *string dbIndex db.DBNum keyLevel int isKey bool + defVal string + tblOwner *bool + hasChildSubTree bool + hasNonTerminalNode bool + subscribePref *string + subscribeOnChg int + subscribeMinIntvl int + cascadeDel int + virtualTbl *bool + nameWithMod *string + xfmrPre string } type dbInfo struct { dbIndex db.DBNum keyName *string fieldType string + rpcFunc string dbEntry *yang.Entry yangXpath []string module string + delim string + leafRefPath []string + listName []string + keyList []string + xfmrValue *string + hasXfmrFn bool + cascadeDel int +} + +type sonicTblSeqnInfo struct { + OrdTbl []string + DepTbl map[string][]string +} + +type mdlInfo struct { + Org string + Ver string } var xYangSpecMap map[string]*yangXpathInfo var xDbSpecMap map[string]*dbInfo var xDbSpecOrdTblMap map[string][]string //map of module-name to ordered list of db tables { "sonic-acl" : ["ACL_TABLE", "ACL_RULE"] } +var xDbSpecTblSeqnMap map[string]*sonicTblSeqnInfo +var xMdlCpbltMap map[string]*mdlInfo +var sonicOrdTblListMap map[string][]string +var sonicLeafRefMap map[string][]string + +/* Add module name to map storing model info for model capabilities */ +func addMdlCpbltEntry(yangMdlNm string) { + if xMdlCpbltMap == nil { + xMdlCpbltMap = make(map[string]*mdlInfo) + } + mdlInfoEntry := new(mdlInfo) + if mdlInfoEntry == nil { + log.Warningf("Memory allocation failure for storing model info for gnmi - module %v", yangMdlNm) + return + } + mdlInfoEntry.Org = "" + mdlInfoEntry.Ver = "" + xMdlCpbltMap[yangMdlNm] = mdlInfoEntry +} + +/* Add version and organization info for model capabilities into map */ +func addMdlCpbltData(yangMdlNm string, version string, organization string) { + if xMdlCpbltMap == nil { + xMdlCpbltMap = make(map[string]*mdlInfo) + } + mdlInfoEntry, ok := xMdlCpbltMap[yangMdlNm] + if ((!ok) || (mdlInfoEntry == nil)) { + mdlInfoEntry = new(mdlInfo) + if mdlInfoEntry == nil { + log.Warningf("Memory allocation failure for storing model info for gnmi - module %v", yangMdlNm) + return + } + } + mdlInfoEntry.Ver = version + mdlInfoEntry.Org = organization + xMdlCpbltMap[yangMdlNm] = mdlInfoEntry +} /* update transformer spec with db-node */ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string) { + if len(tableName) == 0 { + return + } _, ok := xDbSpecMap[tableName] if ok { xDbSpecMap[tableName].yangXpath = append(xDbSpecMap[tableName].yangXpath, xpath) @@ -71,9 +143,59 @@ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string } } +func childContainerListPresenceFlagSet(xpath string) { + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + parXpathData.hasNonTerminalNode = true + } + parXpath = parentXpathGet(parXpath) + } +} + +func childSubTreePresenceFlagSet(xpath string) { + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + parXpathData.hasChildSubTree = true + } + parXpath = parentXpathGet(parXpath) + } +} + /* Recursive api to fill the map with yang details */ -func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string) { +func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string, xpathFull string) { xpath := "" + curKeyLevel := 0 + curXpathFull := "" + + if entry != nil && (entry.Kind == yang.CaseEntry || entry.Kind == yang.ChoiceEntry) { + curXpathFull = xpathFull + "/" + entry.Name + if _, ok := xYangSpecMap[xpathPrefix]; ok { + curKeyLevel = xYangSpecMap[xpathPrefix].keyLevel + } + curXpathData, ok := xYangSpecMap[curXpathFull] + if !ok { + curXpathData = new(yangXpathInfo) + curXpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[curXpathFull] = curXpathData + } + curXpathData.yangDataType = strings.ToLower(yang.EntryKindToName[entry.Kind]) + curXpathData.yangEntry = entry + if xYangSpecMap[xpathPrefix].subscribePref != nil { + curXpathData.subscribePref = xYangSpecMap[xpathPrefix].subscribePref + } + curXpathData.subscribeOnChg = xYangSpecMap[xpathPrefix].subscribeOnChg + curXpathData.subscribeMinIntvl = xYangSpecMap[xpathPrefix].subscribeMinIntvl + curXpathData.cascadeDel = xYangSpecMap[xpathPrefix].cascadeDel + xpath = xpathPrefix + } else { /* create the yang xpath */ if xYangSpecMap[xpathPrefix] != nil && xYangSpecMap[xpathPrefix].yangDataType == "module" { /* module name is separated from the rest of xpath with ":" */ @@ -82,23 +204,41 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr xpath = xpathPrefix + "/" + entry.Name } + updateChoiceCaseXpath := false + curXpathFull = xpath + if xpathPrefix != xpathFull { + curXpathFull = xpathFull + "/" + entry.Name + if annotNode, ok := xYangSpecMap[curXpathFull]; ok { + xpathData := new(yangXpathInfo) + xpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[xpath] = xpathData + copyYangXpathSpecData(xYangSpecMap[xpath], annotNode) + updateChoiceCaseXpath = true + } + } + xpathData, ok := xYangSpecMap[xpath] if !ok { xpathData = new(yangXpathInfo) xYangSpecMap[xpath] = xpathData xpathData.dbIndex = db.ConfigDB // default value + xpathData.subscribeOnChg = XFMR_INVALID + xpathData.subscribeMinIntvl = XFMR_INVALID + xpathData.cascadeDel = XFMR_INVALID } else { - xpathData = xYangSpecMap[xpath] + if len(xpathData.xfmrFunc) > 0 { + childSubTreePresenceFlagSet(xpath) + } } xpathData.yangDataType = entry.Node.Statement().Keyword - if entry.Node.Statement().Keyword == "list" && xpathData.tableName != nil { + if (xpathData.tableName != nil && *xpathData.tableName != "") { childToUpdateParent(xpath, *xpathData.tableName) } parentXpathData, ok := xYangSpecMap[xpathPrefix] /* init current xpath table data with its parent data, change only if needed. */ - if ok { + if ok && xpathData.tableName == nil { if xpathData.tableName == nil && parentXpathData.tableName != nil && xpathData.xfmrTbl == nil { xpathData.tableName = parentXpathData.tableName } else if xpathData.xfmrTbl == nil && parentXpathData.xfmrTbl != nil { @@ -119,20 +259,79 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr xpathData.xfmrFunc = parentXpathData.xfmrFunc } - if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) == 0 { + if ok && (parentXpathData.subscribeMinIntvl == XFMR_INVALID || + parentXpathData.subscribeOnChg == XFMR_INVALID) { + log.Warningf("Susbscribe MinInterval/OnChange flag is set to invalid for(%v) \r\n", xpathPrefix) + return + } + + if ok { + if xpathData.subscribeOnChg == XFMR_INVALID { + xpathData.subscribeOnChg = parentXpathData.subscribeOnChg + } + + if xpathData.subscribeMinIntvl == XFMR_INVALID { + xpathData.subscribeMinIntvl = parentXpathData.subscribeMinIntvl + } + + if xpathData.subscribePref == nil && parentXpathData.subscribePref != nil { + xpathData.subscribePref = parentXpathData.subscribePref + } + + if xpathData.subscribePref != nil && *xpathData.subscribePref == "NONE" { + xpathData.subscribePref = nil + } + + if parentXpathData.cascadeDel == XFMR_INVALID { + /* should not hit this case */ + log.Warningf("Cascade-delete flag is set to invalid for(%v) \r\n", xpathPrefix) + return + } + + if xpathData.cascadeDel == XFMR_INVALID && xpathData.dbIndex == db.ConfigDB{ + xpathData.cascadeDel = parentXpathData.cascadeDel + } + + if entry.Prefix != nil && entry.Prefix.Parent != nil && + entry.Prefix.Parent.Statement().Keyword == "module" && + parentXpathData.yangEntry.Prefix != nil && parentXpathData.yangEntry.Prefix.Parent != nil { + + if (len(parentXpathData.yangEntry.Prefix.Parent.NName()) > 0) && + (len(parentXpathData.yangEntry.Prefix.Parent.Statement().Keyword) > 0) && + (parentXpathData.yangEntry.Prefix.Parent.NName() != entry.Prefix.Parent.NName()) { + xpathData.nameWithMod = new(string) + *xpathData.nameWithMod = entry.Prefix.Parent.NName() + ":" + entry.Name + } + + } + } + + if ((xpathData.yangDataType == YANG_LEAF || xpathData.yangDataType == YANG_LEAF_LIST) && (len(xpathData.fieldName) == 0)) { + + if len(xpathData.xfmrField) != 0 { + xpathData.xfmrFunc = "" + } if xpathData.tableName != nil && xDbSpecMap[*xpathData.tableName] != nil { - if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[entry.Name] != nil { + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + entry.Name]; ok { xpathData.fieldName = entry.Name - } else if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[strings.ToUpper(entry.Name)] != nil { - xpathData.fieldName = strings.ToUpper(entry.Name) + } else { + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + strings.ToUpper(entry.Name)]; ok { + xpathData.fieldName = strings.ToUpper(entry.Name) + } + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + entry.Name]; ok { + xpathData.fieldName = entry.Name + } } } else if xpathData.xfmrTbl != nil { /* table transformer present */ xpathData.fieldName = entry.Name } } + if xpathData.yangDataType == YANG_LEAF && len(entry.Default) > 0 { + xpathData.defVal = entry.Default + } - if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) > 0 && xpathData.tableName != nil { + if (xpathData.yangDataType == YANG_LEAF || xpathData.yangDataType == YANG_LEAF_LIST) && len(xpathData.fieldName) > 0 && xpathData.tableName != nil { dbPath := *xpathData.tableName + "/" + xpathData.fieldName if xDbSpecMap[dbPath] != nil { xDbSpecMap[dbPath].yangXpath = append(xDbSpecMap[dbPath].yangXpath, xpath) @@ -148,8 +347,11 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr keyXpath := make([]string, len(strings.Split(entry.Key, " "))) for id, keyName := range(strings.Split(entry.Key, " ")) { keyXpath[id] = xpath + "/" + keyName - keyXpathData := new(yangXpathInfo) - xYangSpecMap[xpath + "/" + keyName] = keyXpathData + if _, ok := xYangSpecMap[xpath + "/" + keyName]; !ok { + keyXpathData := new(yangXpathInfo) + keyXpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[xpath + "/" + keyName] = keyXpathData + } xYangSpecMap[xpath + "/" + keyName].isKey = true } @@ -165,6 +367,33 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr } else if parentXpathData != nil && parentXpathData.keyXpath != nil { xpathData.keyXpath = parentXpathData.keyXpath } + xpathData.yangEntry = entry + + if xpathData.subscribeMinIntvl == XFMR_INVALID { + xpathData.subscribeMinIntvl = 0 + } + + if xpathData.subscribeOnChg == XFMR_INVALID { + xpathData.subscribeOnChg = XFMR_ENABLE + } + if ((xpathData.subscribePref != nil) && (*xpathData.subscribePref == "onchange") && (xpathData.subscribeOnChg == XFMR_DISABLE)) { + log.Infof("subscribe OnChange is disabled so setting subscribe preference to default/sample from onchange for xpath - %v", xpath) + xpathData.subscribePref = nil + } + if xpathData.cascadeDel == XFMR_INVALID { + /* set to default value */ + xpathData.cascadeDel = XFMR_DISABLE + } + + if updateChoiceCaseXpath { + copyYangXpathSpecData(xYangSpecMap[curXpathFull], xYangSpecMap[xpath]) + } + + if xpathData.yangDataType == YANG_CONTAINER || xpathData.yangDataType == YANG_LIST { + childContainerListPresenceFlagSet(xpath) + } + + } /* get current obj's children */ var childList []string @@ -172,10 +401,9 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr childList = append(childList, k) } - xpathData.yangEntry = entry /* now recurse, filling the map with current node's children info */ for _, child := range childList { - yangToDbMapFill(curKeyLevel, xYangSpecMap, entry.Dir[child], xpath) + yangToDbMapFill(curKeyLevel, xYangSpecMap, entry.Dir[child], xpath, curXpathFull) } } @@ -196,27 +424,78 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { /* Start to fill xpath based map with yang data */ keyLevel := 0 - yangToDbMapFill(keyLevel, xYangSpecMap, e, "") + yangToDbMapFill(keyLevel, xYangSpecMap, e, "", "") // Fill the ordered map of child tables list for oc yangs updateSchemaOrderedMap(module, e) } - mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") - dbMapPrint("/tmp/dbSpecMap.txt") + + //sonicOrdTblListMap = make(map[string][]string) + //jsonfile := YangPath + TblInfoJsonFile + //xlateJsonTblInfoLoad(sonicOrdTblListMap, jsonfile) + + mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") + dbMapPrint("/tmp/dbSpecMap.txt") + xDbSpecTblSeqnMapPrint("/tmp/dbSpecTblSeqnMap.txt") + sonicLeafRefDataPrint("/tmp/sonicLeafRef.log") + sonicLeafRefMap = nil +} + +func dbSpecXpathGet(inPath string) (string, error){ + /* This api currently handles only cointainer inside a list for sonic-yang. + Should be enhanced to support nested list in future. */ + var err error + specPath := "" + + xpath, _, err := XfmrRemoveXPATHPredicates(inPath) + if err != nil { + log.Warningf("xpath conversion failed for(%v) \r\n", inPath) + return specPath, err + } + + pathList := strings.Split(xpath, "/") + if len(pathList) < 3 { + log.Warningf("Leaf-ref path not valid(%v) \r\n", inPath) + return specPath, err + } + + tableName := pathList[2] + fieldName := pathList[len(pathList)-1] + specPath = tableName+"/"+fieldName + return specPath, err +} + +/* Convert a relative path to an absolute path */ +func relativeToActualPathGet(relPath string, entry *yang.Entry) string { + actDbSpecPath := "" + xpath, _, err := XfmrRemoveXPATHPredicates(relPath) + if err != nil { + return actDbSpecPath + } + + pathList := strings.Split(xpath[1:], "/") + if len(pathList) > 3 && (pathList[len(pathList)-3] != "..") { + tableName := pathList[len(pathList)-3] + fieldName := pathList[len(pathList)-1] + actDbSpecPath = tableName+"/"+fieldName + } + return actDbSpecPath } /* Fill the map with db details */ -func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { +func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { + if entry == nil { + return + } + entryType := entry.Node.Statement().Keyword if entry.Name != moduleNm { - if entryType == "container" { + if entryType == "container" || entryType == "rpc" { tableName = entry.Name } - - if !isYangResType(entryType) { dbXpath := tableName - if entryType != "container" { + if entryType != "container" && entryType != "rpc" { dbXpath = tableName + "/" + entry.Name } xDbSpecMap[dbXpath] = new(dbInfo) @@ -224,6 +503,7 @@ func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap[dbXpath].dbEntry = entry xDbSpecMap[dbXpath].fieldType = entryType xDbSpecMap[dbXpath].module = moduleNm + xDbSpecMap[dbXpath].cascadeDel = XFMR_INVALID if entryType == "container" { xDbSpecMap[dbXpath].dbIndex = db.ConfigDB if entry.Exts != nil && len(entry.Exts) > 0 { @@ -236,35 +516,112 @@ func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap[dbXpath].keyName = new(string) } *xDbSpecMap[dbXpath].keyName = ext.NName() + case "rpc-callback" : + xDbSpecMap[dbXpath].rpcFunc = ext.NName() + case "db-name" : + xDbSpecMap[dbXpath].dbIndex = dbNameToIndex(ext.NName()) + case "key-delim" : + xDbSpecMap[dbXpath].delim = ext.NName() default : log.Infof("Unsupported ext type(%v) for xpath(%v).", tagType, dbXpath) } } } + } else if tblSpecInfo, ok := xDbSpecMap[tableName]; ok && (entryType == YANG_LIST && len(entry.Key) != 0) { + tblSpecInfo.listName = append(tblSpecInfo.listName, entry.Name) + xDbSpecMap[dbXpath].keyList = append(xDbSpecMap[dbXpath].keyList, strings.Split(entry.Key, " ")...) + } else if entryType == YANG_LEAF || entryType == YANG_LEAF_LIST { + if entry.Type.Kind == yang.Yleafref { + var lerr error + lrefpath := entry.Type.Path + if (strings.Contains(lrefpath, "..")) { + lrefpath = relativeToActualPathGet(lrefpath, entry) + } else { + lrefpath, lerr = dbSpecXpathGet(lrefpath) + if lerr != nil { + log.Warningf("Failed to add leaf-ref for(%v) \r\n", entry.Type.Path) + return + } + + } + xDbSpecMap[dbXpath].leafRefPath = append(xDbSpecMap[dbXpath].leafRefPath, lrefpath) + sonicLeafRefMap[lrefpath] = append(sonicLeafRefMap[lrefpath], dbXpath) + } else if entry.Type.Kind == yang.Yunion && len(entry.Type.Type) > 0 { + for _, ltype := range entry.Type.Type { + if ltype.Kind == yang.Yleafref { + var lerr error + lrefpath := ltype.Path + if (strings.Contains(lrefpath, "..")) { + lrefpath = relativeToActualPathGet(lrefpath, entry) + } else { + lrefpath, lerr = dbSpecXpathGet(lrefpath) + if lerr != nil { + log.Warningf("Failed to add leaf-ref for(%v) \r\n", ltype.Path) + return + } + + } + xDbSpecMap[dbXpath].leafRefPath = append(xDbSpecMap[dbXpath].leafRefPath, lrefpath) + sonicLeafRefMap[lrefpath] = append(sonicLeafRefMap[lrefpath], dbXpath) + } + } + } } - } } else { moduleXpath := "/" + moduleNm + ":" + entry.Name xDbSpecMap[moduleXpath] = new(dbInfo) xDbSpecMap[moduleXpath].dbEntry = entry xDbSpecMap[moduleXpath].fieldType = entryType xDbSpecMap[moduleXpath].module = moduleNm - } + xDbSpecMap[moduleXpath].cascadeDel = XFMR_INVALID + for { + done := true + sncTblInfo := new(sonicTblSeqnInfo) + if sncTblInfo == nil { + log.Warningf("Memory allocation failure for storing Tbl order and dependency info for sonic module %v", moduleNm) + break + } + cvlSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + log.Warningf("Failure in creating CVL validation session object required to use CVl API to get Tbl info for module %v - %v", moduleNm, cvlRetSess) + break + } + var cvlRetOrdTbl cvl.CVLRetCode + sncTblInfo.OrdTbl, cvlRetOrdTbl = cvlSess.GetOrderedTables(moduleNm) + if cvlRetOrdTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.GetOrderedTables(%v) - %v", moduleNm, cvlRetOrdTbl) + + } + sncTblInfo.DepTbl = make(map[string][]string) + if sncTblInfo.DepTbl == nil { + log.Warningf("sncTblInfo.DepTbl is nill , no space to store dependency table list for sonic module %v", moduleNm) + cvl.ValidationSessClose(cvlSess) + break + } + for _, tbl := range(sncTblInfo.OrdTbl) { + var cvlRetDepTbl cvl.CVLRetCode + sncTblInfo.DepTbl[tbl], cvlRetDepTbl = cvlSess.GetDepTables(moduleNm, tbl) + if cvlRetDepTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.GetDepTables(%v, %v) - %v", moduleNm, tbl, cvlRetDepTbl) + } - var childList []string - for _, k := range entry.DirOKeys { - childList = append(childList, k) - } - if entryType == "container" && trkTpCnt { - xDbSpecOrdTblMap[moduleNm] = childList - log.Info("xDbSpecOrdTblMap after appending ", xDbSpecOrdTblMap) - trkTpCnt = false + } + xDbSpecTblSeqnMap[moduleNm] = sncTblInfo + cvl.ValidationSessClose(cvlSess) + if done { + break + } + } + } + var childList []string + childList = append(childList, entry.DirOKeys...) + for _, child := range childList { childPath := tableName + "/" + entry.Dir[child].Name - dbMapFill(tableName, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) + dbMapFill(tableName, childPath, moduleNm, xDbSpecMap, entry.Dir[child]) } } @@ -275,15 +632,15 @@ func dbMapBuild(entries []*yang.Entry) { } xDbSpecMap = make(map[string]*dbInfo) xDbSpecOrdTblMap = make(map[string][]string) + xDbSpecTblSeqnMap = make(map[string]*sonicTblSeqnInfo) + sonicLeafRefMap = make(map[string][]string) for _, e := range entries { if e == nil || len(e.Dir) == 0 { continue } moduleNm := e.Name - log.Infof("Module name(%v)", moduleNm) - trkTpCnt := true - dbMapFill("", "", moduleNm, trkTpCnt, xDbSpecMap, e) + dbMapFill("", "", moduleNm, xDbSpecMap, e) } } @@ -297,25 +654,55 @@ func childToUpdateParent( xpath string, tableName string) { _, ok := xYangSpecMap[parent] if !ok { xpathData = new(yangXpathInfo) + xpathData.dbIndex = db.ConfigDB // default value xYangSpecMap[parent] = xpathData } - xYangSpecMap[parent].childTable = append(xYangSpecMap[parent].childTable, tableName) - if xYangSpecMap[parent].yangEntry != nil && - xYangSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { + + parentXpathData := xYangSpecMap[parent] + if !contains(parentXpathData.childTable, tableName) { + parentXpathData.childTable = append(parentXpathData.childTable, tableName) + } + + if parentXpathData.yangEntry != nil && parentXpathData.yangEntry.Node.Statement().Keyword == "list" && + (parentXpathData.tableName != nil || parentXpathData.xfmrTbl != nil) { return } childToUpdateParent(parent, tableName) } +func dbNameToIndex(dbName string) db.DBNum { + dbIndex := db.ConfigDB + switch dbName { + case "APPL_DB" : + dbIndex = db.ApplDB + case "ASIC_DB" : + dbIndex = db.AsicDB + case "COUNTERS_DB" : + dbIndex = db.CountersDB + case "LOGLEVEL_DB" : + dbIndex = db.LogLevelDB + case "CONFIG_DB" : + dbIndex = db.ConfigDB + case "FLEX_COUNTER_DB" : + dbIndex = db.FlexCounterDB + case "STATE_DB" : + dbIndex = db.StateDB + case "ERROR_DB" : + dbIndex = db.ErrorDB + case "USER_DB" : + dbIndex = db.UserDB + } + return dbIndex +} + /* Build lookup map based on yang xpath */ func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry *yang.Entry) { xpathData := new(yangXpathInfo) - _, ok := xYangSpecMap[xpath] - if !ok { - fmt.Printf("Xpath not found(%v) \r\n", xpath) - } xpathData.dbIndex = db.ConfigDB // default value + xpathData.subscribeOnChg = XFMR_INVALID + xpathData.subscribeMinIntvl = XFMR_INVALID + xpathData.cascadeDel = XFMR_INVALID /* fill table with yang extension data. */ if entry != nil && len(entry.Exts) > 0 { for _, ext := range entry.Exts { @@ -347,30 +734,62 @@ func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry case "key-delimiter" : xpathData.delim = ext.NName() case "field-transformer" : - xpathData.xfmrFunc = ext.NName() + xpathData.xfmrField = ext.NName() case "post-transformer" : xpathData.xfmrPost = ext.NName() + case "pre-transformer" : + xpathData.xfmrPre = ext.NName() case "get-validate" : xpathData.validateFunc = ext.NName() + case "rpc-callback" : + xpathData.rpcFunc = ext.NName() case "use-self-key" : xpathData.keyXpath = nil case "db-name" : - if ext.NName() == "APPL_DB" { - xpathData.dbIndex = db.ApplDB - } else if ext.NName() == "ASIC_DB" { - xpathData.dbIndex = db.AsicDB - } else if ext.NName() == "COUNTERS_DB" { - xpathData.dbIndex = db.CountersDB - } else if ext.NName() == "LOGLEVEL_DB" { - xpathData.dbIndex = db.LogLevelDB - } else if ext.NName() == "CONFIG_DB" { - xpathData.dbIndex = db.ConfigDB - } else if ext.NName() == "FLEX_COUNTER_DB" { - xpathData.dbIndex = db.FlexCounterDB - } else if ext.NName() == "STATE_DB" { - xpathData.dbIndex = db.StateDB + xpathData.dbIndex = dbNameToIndex(ext.NName()) + case "table-owner" : + if xpathData.tblOwner == nil { + xpathData.tblOwner = new(bool) + *xpathData.tblOwner = true + } + if strings.EqualFold(ext.NName(), "False") { + *xpathData.tblOwner = false + } + case "subscribe-preference" : + if xpathData.subscribePref == nil { + xpathData.subscribePref = new(string) + } + *xpathData.subscribePref = ext.NName() + case "subscribe-on-change" : + if ext.NName() == "enable" || ext.NName() == "ENABLE" { + xpathData.subscribeOnChg = XFMR_ENABLE + } else { + xpathData.subscribeOnChg = XFMR_DISABLE + } + case "subscribe-min-interval" : + if ext.NName() == "NONE" { + xpathData.subscribeMinIntvl = 0 } else { - xpathData.dbIndex = db.ConfigDB + minIntvl, err := strconv.Atoi(ext.NName()) + if err != nil { + log.Warningf("Invalid subscribe min interval time(%v).\r\n", ext.NName()) + return + } + xpathData.subscribeMinIntvl = minIntvl + } + case "cascade-delete" : + if ext.NName() == "ENABLE" || ext.NName() == "enable" { + xpathData.cascadeDel = XFMR_ENABLE + } else { + xpathData.cascadeDel = XFMR_DISABLE + } + case "virtual-table" : + if xpathData.virtualTbl == nil { + xpathData.virtualTbl = new(bool) + *xpathData.virtualTbl = false + } + if strings.EqualFold(ext.NName(), "True") { + *xpathData.virtualTbl = true } } } @@ -419,19 +838,45 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya var dbXpathData *dbInfo var ok bool - //Currently sonic-yang annotation is supported for "list" type only. - listName := strings.Split(dbXpath, "/") - if len(listName) < 3 { - log.Errorf("Invalid list xpath length(%v) \r\n", dbXpath) - return err + pname := strings.Split(dbXpath, "/") + if len(pname) < 3 { + // check rpc? + rpcName := strings.Split(pname[1], ":") + if len(rpcName) < 2 { + log.Warningf("DB spec-map data not found(%v) \r\n", rpcName) + return err + } + dbXpathData, ok = xDbSpecMap[rpcName[1]] + if ok && dbXpathData.fieldType == "rpc" { + if entry != nil && len(entry.Exts) > 0 { + for _, ext := range entry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + switch tagType { + case "rpc-callback" : + dbXpathData.rpcFunc = ext.NName() + default : + } + } + } + return err + } else { + log.Warningf("DB spec-map data not found(%v) \r\n", dbXpath) + return err + } } - dbXpathData, ok = xDbSpecMap[listName[2]] + + tableName := pname[2] + // container(redis tablename) + dbXpathData, ok = xDbSpecMap[tableName] if !ok { - log.Errorf("DB spec-map data not found(%v) \r\n", dbXpath) + log.Warningf("DB spec-map data not found(%v) \r\n", dbXpath) return err } - log.Infof("Annotate dbSpecMap for (%v)(listName:%v)\r\n", dbXpath, listName[2]) - dbXpathData.dbIndex = db.ConfigDB // default value + + if dbXpathData.dbIndex >= db.MaxDB { + dbXpathData.dbIndex = db.ConfigDB // default value + } /* fill table with cvl yang extension data. */ if entry != nil && len(entry.Exts) > 0 { @@ -445,29 +890,37 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya } *dbXpathData.keyName = ext.NName() case "db-name" : - if ext.NName() == "APPL_DB" { - dbXpathData.dbIndex = db.ApplDB - } else if ext.NName() == "ASIC_DB" { - dbXpathData.dbIndex = db.AsicDB - } else if ext.NName() == "COUNTERS_DB" { - dbXpathData.dbIndex = db.CountersDB - } else if ext.NName() == "LOGLEVEL_DB" { - dbXpathData.dbIndex = db.LogLevelDB - } else if ext.NName() == "CONFIG_DB" { - dbXpathData.dbIndex = db.ConfigDB - } else if ext.NName() == "FLEX_COUNTER_DB" { - dbXpathData.dbIndex = db.FlexCounterDB - } else if ext.NName() == "STATE_DB" { - dbXpathData.dbIndex = db.StateDB + dbXpathData.dbIndex = dbNameToIndex(ext.NName()) + case "value-transformer" : + fieldName := pname[len(pname) - 1] + fieldXpath := tableName+"/"+fieldName + if fldXpathData, ok := xDbSpecMap[fieldXpath]; ok { + fldXpathData.xfmrValue = new(string) + *fldXpathData.xfmrValue = ext.NName() + dbXpathData.hasXfmrFn = true + if xpathList, ok := sonicLeafRefMap[fieldXpath]; ok { + for _, curpath := range(xpathList) { + if curSpecData, ok := xDbSpecMap[curpath]; ok && curSpecData.xfmrValue == nil { + curSpecData.xfmrValue = fldXpathData.xfmrValue + curTableName := strings.Split(curpath, "/")[0] + if curTblSpecInfo, ok := xDbSpecMap[curTableName]; ok { + curTblSpecInfo.hasXfmrFn = true + } + } + } + } + } + case "cascade-delete" : + if ext.NName() == "ENABLE" || ext.NName() == "enable" { + dbXpathData.cascadeDel = XFMR_ENABLE } else { - dbXpathData.dbIndex = db.ConfigDB + dbXpathData.cascadeDel = XFMR_DISABLE } default : } } } - dbMapPrint("/tmp/dbSpecMapFull.txt") return err } @@ -490,6 +943,7 @@ func annotDbSpecMap(annotEntries []*yang.Entry) { } } } + dbMapPrint("/tmp/dbSpecMapFull.txt") } /* Debug function to print the yang xpath lookup map */ @@ -504,10 +958,32 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") fmt.Fprintf(fp, "%v:\r\n", k) fmt.Fprintf(fp, " yangDataType: %v\r\n", d.yangDataType) + if d.nameWithMod != nil { + fmt.Fprintf(fp, " nameWithMod : %v\r\n", *d.nameWithMod) + } + fmt.Fprintf(fp, " cascadeDel : %v\r\n", d.cascadeDel) + fmt.Fprintf(fp, " hasChildSubTree : %v\r\n", d.hasChildSubTree) + fmt.Fprintf(fp, " hasNonTerminalNode : %v\r\n", d.hasNonTerminalNode) + fmt.Fprintf(fp, " subscribeOnChg : %v\r\n", d.subscribeOnChg) + fmt.Fprintf(fp, " subscribeMinIntvl : %v\r\n", d.subscribeMinIntvl) + if d.subscribePref != nil { + fmt.Fprintf(fp, " subscribePref : %v\r\n", *d.subscribePref) + } + fmt.Fprintf(fp, " hasChildSubTree: %v\r\n", d.hasChildSubTree) fmt.Fprintf(fp, " tableName: ") if d.tableName != nil { fmt.Fprintf(fp, "%v", *d.tableName) } + fmt.Fprintf(fp, "\r\n postXfmr : %v", d.xfmrPost) + fmt.Fprintf(fp, "\r\n tblOwner: ") + if d.tblOwner != nil { + fmt.Fprintf(fp, "%v", *d.tblOwner) + } + fmt.Fprintf(fp, "\r\n virtualTbl: ") + if d.virtualTbl != nil { + fmt.Fprintf(fp, "%v", *d.virtualTbl) + } + fmt.Fprintf(fp, "\r\n preXfmr : %v", d.xfmrPre) fmt.Fprintf(fp, "\r\n xfmrTbl : ") if d.xfmrTbl != nil { fmt.Fprintf(fp, "%v", *d.xfmrTbl) @@ -518,11 +994,14 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { } fmt.Fprintf(fp, "\r\n childTbl : %v", d.childTable) fmt.Fprintf(fp, "\r\n FieldName: %v", d.fieldName) + fmt.Fprintf(fp, "\r\n defVal : %v", d.defVal) fmt.Fprintf(fp, "\r\n keyLevel : %v", d.keyLevel) fmt.Fprintf(fp, "\r\n xfmrKeyFn: %v", d.xfmrKey) fmt.Fprintf(fp, "\r\n xfmrFunc : %v", d.xfmrFunc) + fmt.Fprintf(fp, "\r\n xfmrField :%v", d.xfmrField) fmt.Fprintf(fp, "\r\n dbIndex : %v", d.dbIndex) fmt.Fprintf(fp, "\r\n validateFunc : %v", d.validateFunc) + fmt.Fprintf(fp, "\r\n rpcFunc : %v", d.rpcFunc) fmt.Fprintf(fp, "\r\n yangEntry: ") if d.yangEntry != nil { fmt.Fprintf(fp, "%v", *d.yangEntry) @@ -550,21 +1029,89 @@ func dbMapPrint( fname string) { defer fp.Close() fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") for k, v := range xDbSpecMap { - fmt.Fprintf(fp, " field:%v \r\n", k) + fmt.Fprintf(fp, " field:%v: \r\n", k) fmt.Fprintf(fp, " type :%v \r\n", v.fieldType) fmt.Fprintf(fp, " db-type :%v \r\n", v.dbIndex) + fmt.Fprintf(fp, " rpcFunc :%v \r\n", v.rpcFunc) + fmt.Fprintf(fp, " hasXfmrFn:%v \r\n", v.hasXfmrFn) fmt.Fprintf(fp, " module :%v \r\n", v.module) + fmt.Fprintf(fp, " listName :%v \r\n", v.listName) + fmt.Fprintf(fp, " keyList :%v \r\n", v.keyList) + fmt.Fprintf(fp, " cascadeDel :%v \r\n", v.cascadeDel) + if v.xfmrValue != nil { + fmt.Fprintf(fp, " xfmrValue:%v \r\n", *v.xfmrValue) + } + fmt.Fprintf(fp, " leafRefPath:%v \r\n", v.leafRefPath) fmt.Fprintf(fp, " KeyName: ") if v.keyName != nil { - fmt.Fprintf(fp, "%v", *v.keyName) + fmt.Fprintf(fp, "%v\r\n", *v.keyName) } - fmt.Fprintf(fp, "\r\n oc-yang :%v \r\n", v.yangXpath) - fmt.Fprintf(fp, " cvl-yang :%v \r\n", v.dbEntry) + for _, yxpath := range v.yangXpath { + fmt.Fprintf(fp, "\r\n oc-yang :%v ", yxpath) + } + fmt.Fprintf(fp, "\r\n cvl-yang :%v \r\n", v.dbEntry) fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") } } +func xDbSpecTblSeqnMapPrint(fname string) { + fp, err := os.Create(fname) + if err != nil { + return + } + defer fp.Close() + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + if xDbSpecTblSeqnMap == nil { + return + } + for mdlNm, mdlTblSeqnDt := range xDbSpecTblSeqnMap { + fmt.Fprintf(fp, "%v : { \r\n", mdlNm) + if mdlTblSeqnDt == nil { + continue + } + fmt.Fprintf(fp, " OrderedTableList : %v\r\n", mdlTblSeqnDt.OrdTbl) + fmt.Fprintf(fp, " Dependent table list : {\r\n") + if mdlTblSeqnDt.DepTbl == nil { + fmt.Fprintf(fp, " }\r\n") + fmt.Fprintf(fp, "}\r\n") + continue + } + for tblNm, DepTblLst := range mdlTblSeqnDt.DepTbl { + fmt.Fprintf(fp, " %v : %v\r\n", tblNm, DepTblLst) + } + fmt.Fprintf(fp, " }\r\n") + fmt.Fprintf(fp, "}\r\n") + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + fmt.Fprintf (fp, "OrderedTableList from json: \r\n") + + for tbl, tlist := range sonicOrdTblListMap { + fmt.Fprintf(fp, " %v : %v\r\n", tbl, tlist) + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + +} + +func sonicLeafRefDataPrint(fname string) { + fp, err := os.Create(fname) + if err != nil { + return + } + defer fp.Close() + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + if xDbSpecTblSeqnMap == nil { + return + } + for lref, data := range sonicLeafRefMap { + fmt.Fprintf (fp, "leafref: %v\r\n", lref) + for i, d := range data { + fmt.Fprintf (fp, " (%v) %v\r\n", i, d) + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + } +} + func updateSchemaOrderedMap(module string, entry *yang.Entry) { var children []string if entry.Node.Statement().Keyword == "module" { diff --git a/translib/utils/utils.go b/translib/utils/utils.go new file mode 100644 index 000000000000..466c618bf979 --- /dev/null +++ b/translib/utils/utils.go @@ -0,0 +1,71 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + + +package utils + +import ( + //"github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/cvl" + "fmt" + log "github.com/golang/glog" +) + +// SortAsPerTblDeps - sort transformer result table list based on dependencies (using CVL API) tables to be used for CRUD operations +func SortAsPerTblDeps(tblLst []string) ([]string, error) { + var resultTblLst []string + var err error + logStr := "Failure in CVL API to sort table list as per dependencies." + + cvSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + + log.Errorf("Failure in creating CVL validation session object required to use CVl API(sort table list as per dependencies) - %v", cvlRetSess) + err = fmt.Errorf("%v", logStr) + return resultTblLst, err + } + cvlSortDepTblList, cvlRetDepTbl := cvSess.SortDepTables(tblLst) + if cvlRetDepTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.SortDepTables: %v", cvlRetDepTbl) + cvl.ValidationSessClose(cvSess) + err = fmt.Errorf("%v", logStr) + return resultTblLst, err + } + log.Info("cvlSortDepTblList = ", cvlSortDepTblList) + resultTblLst = cvlSortDepTblList + + cvl.ValidationSessClose(cvSess) + return resultTblLst, err + +} + +// RemoveElement - Remove a specific string from a list of strings +func RemoveElement(sl []string, str string) []string { + for i := 0; i < len(sl); i++ { + if sl[i] == str { + sl = append(sl[:i], sl[i+1:]...) + i-- + break + } + } + return sl +} + + +