Skip to content

Commit

Permalink
* [saidump]
Browse files Browse the repository at this point in the history 
•	Saidump for DNX-SAI sonic-net/sonic-buildimage#13561

Solution and modification:
To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it.

(1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host.
(2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format.
(3) Add a new script file: files/scripts/saidump.sh, to do the below steps
  For each ASIC0, such as ASIC0,

  1. Save the Redis data.
  sudo sonic-db-cli -n asic$1 SAVE > /dev/null

  2. Move dump files to /var/run/redisX/
  docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/"

  3. Run rdb command to convert the dump files into JSON files
  sudo python /usr/local/bin/rdb --command json  /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null

  4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output.
  docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json"

  5. clear
  sudo rm -f /var/run/redis$1/dump.rdb
  sudo rm -f /var/run/redis$1/dump.json

(4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh
  • Loading branch information
@JunhongMao
JunhongMao committed Sep 8, 2023
1 parent a94b8e7 commit 11dd735
Showing 1 changed file with 56 additions and 19 deletions.
75 changes: 56 additions & 19 deletions saidump/saidump.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <iostream>
#include <fstream>
#include <regex>
#include <climits>

extern "C" {
#include <sai.h>
Expand All @@ -22,31 +23,35 @@ extern "C" {
using namespace swss;
using json = nlohmann::json;

// 100 MB
constexpr int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100;
// Default value: 100 MB
constexpr int64_t RDB_JSON_MAX_SIZE = 1024 * 1024 * 100;

struct CmdOptions
{
bool skipAttributes;
bool dumpTempView;
bool dumpGraph;
std::string rdbFile;
std::string rdbJsonFile;
uint64_t rdbJSonSizeLimit;
};


static CmdOptions g_cmdOptions;
static std::map<sai_object_id_t, const TableMap*> g_oid_map;

void printUsage()
{
SWSS_LOG_ENTER();

std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl;
std::cout << "Usage: saidump [-t] [-g] [-r] [-m] [-h]" << std::endl;
std::cout << " -t --tempView:" << std::endl;
std::cout << " Dump temp view" << std::endl;
std::cout << " -g --dumpGraph:" << std::endl;
std::cout << " Dump current graph" << std::endl;
std::cout << " -r --rdb:" << std::endl;
std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command" << std::endl;
std::cout << " Dump by parsing the RDB JSON file, which is created by rdbtools based on Redis dump.rdb that is generated by Redis SAVE command" << std::endl;
std::cout << " -m --max:" << std::endl;
std::cout << " Config the the RDB JSON file's max size in MB, which is optional with default value 100MB" << std::endl;
std::cout << " -h --help:" << std::endl;
std::cout << " Print out this message" << std::endl;
}
Expand All @@ -60,7 +65,8 @@ CmdOptions handleCmdLine(int argc, char **argv)
options.dumpTempView = false;
options.dumpGraph = false;

const char* const optstring = "gtr:h";
const char* const optstring = "gtr:m:h";
uint64_t result = 0;

while (true)
{
Expand All @@ -69,6 +75,7 @@ CmdOptions handleCmdLine(int argc, char **argv)
{ "dumpGraph", no_argument, 0, 'g' },
{ "tempView", no_argument, 0, 't' },
{ "rdb", required_argument, 0, 'r' },
{ "max", required_argument, 0, 'm' },
{ "help", no_argument, 0, 'h' },
{ 0, 0, 0, 0 }
};
Expand Down Expand Up @@ -96,7 +103,36 @@ CmdOptions handleCmdLine(int argc, char **argv)

case 'r':
SWSS_LOG_NOTICE("Dumping from %s", optarg);
options.rdbFile = std::string(optarg);
options.rdbJsonFile = std::string(optarg);
break;

case 'm':
if(!regex_match(optarg, std::regex(R"([+]?\d+)"))) //only positive numeric chars are valid, such as 3984, +3232, etc.
{
SWSS_LOG_WARN("invalid option -m %s", optarg);
printUsage();
exit(EXIT_SUCCESS);
}

result = strtoull(optarg, NULL, 0);

if((errno == ERANGE && (result == ULLONG_MAX || result == 0)) || (errno != 0 && result == 0) || result >= INT_MAX)
{
SWSS_LOG_WARN("invalid option -m %s", optarg);
printUsage();
exit(EXIT_SUCCESS);
}

if(result == 0)
{
options.rdbJSonSizeLimit = RDB_JSON_MAX_SIZE;
}
else
{
options.rdbJSonSizeLimit = result * 1024 * 1024;
}

SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024);
break;

case 'h':
Expand Down Expand Up @@ -419,9 +455,9 @@ void dumpGraph(const TableDump& td)
#define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); }

/**
* @brief Preprocess the input json file to make sure it's a valid json file.
* @brief Preprocess the input JSON file to make sure it's a valid JSON file for Nlohmann JSON library.
*/
static sai_status_t preProcessFile(std::string file_name)
static sai_status_t preProcessFile(const std::string file_name)
{
SWSS_LOG_ENTER();

Expand All @@ -434,11 +470,11 @@ static sai_status_t preProcessFile(std::string file_name)
}

input_file.seekg(0, std::ios::end); // Move to the end of the file
int64_t file_size = input_file.tellg(); // Get the current position
uint64_t file_size = input_file.tellg(); // Get the current position

if (file_size >= RDB_FILE_MAX_SIZE)
if (file_size >= g_cmdOptions.rdbJSonSizeLimit)
{
SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024);
SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024);
return SAI_STATUS_FAILURE;
}

Expand Down Expand Up @@ -471,15 +507,15 @@ static sai_status_t preProcessFile(std::string file_name)
return SAI_STATUS_SUCCESS;
}

static sai_status_t dumpFromRedisRdbFile(std::string file_name)
static sai_status_t dumpFromRedisRdbJson(const std::string file_name)
{
SWSS_LOG_ENTER();

std::ifstream input_file(file_name);

if (!input_file.is_open())
{
SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str());
SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from Redis RDB JSON file.", file_name.c_str());
return SAI_STATUS_FAILURE;
}

Expand Down Expand Up @@ -539,9 +575,9 @@ static sai_status_t dumpFromRedisRdbFile(std::string file_name)
map[itt.key()] = itt.value();
}

size_t indent = 4;
constexpr size_t LINE_IDENT = 4;
size_t max_len = get_max_attr_len(map);
std::string str_indent = pad_string("", indent);
std::string str_indent = pad_string("", LINE_IDENT);

for (const auto&field: map)
{
Expand Down Expand Up @@ -575,14 +611,15 @@ int main(int argc, char **argv)

g_cmdOptions = handleCmdLine(argc, argv);

if (g_cmdOptions.rdbFile.size() > 0)

if (g_cmdOptions.rdbJsonFile.size() > 0)
{
if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbFile))
if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbJsonFile))
{
return EXIT_FAILURE;
}

return dumpFromRedisRdbFile(g_cmdOptions.rdbFile);
return dumpFromRedisRdbJson(g_cmdOptions.rdbJsonFile);
}

swss::DBConnector db("ASIC_DB", 0);
Expand Down

0 comments on commit 11dd735

Please sign in to comment.