Skip to content

Commit

Permalink
disable meta upgrade from v1 (#3798) (#3827)
Browse files Browse the repository at this point in the history
  • Loading branch information
@darionyaphet
darionyaphet committed Jan 26, 2022
1 parent caf0815 commit ed1f9b1
Show file tree
Hide file tree
Showing 18 changed files with 2 additions and 1,082 deletions.
2 changes: 0 additions & 2 deletions src/daemons/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ nebula_add_executable(
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:meta_v2_thrift_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
${common_deps}
${storage_meta_deps}
LIBRARIES
Expand Down Expand Up @@ -243,7 +242,6 @@ nebula_add_executable(
$<TARGET_OBJECTS:meta_http_handler>
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
${storage_meta_deps}
${common_deps}
LIBRARIES
Expand Down
9 changes: 2 additions & 7 deletions src/daemons/MetaDaemonInit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,13 +145,8 @@ std::unique_ptr<nebula::kvstore::KVStore> initKV(std::vector<nebula::HostAddr> p
LOG(ERROR) << "Meta version is invalid";
return nullptr;
} else if (version == nebula::meta::MetaVersion::V1) {
auto ret = nebula::meta::MetaVersionMan::updateMetaV1ToV2(engine);
if (!ret.ok()) {
LOG(ERROR) << "Update meta from V1 to V2 failed " << ret;
return nullptr;
}

nebula::meta::MetaVersionMan::setMetaVersionToKV(engine, nebula::meta::MetaVersion::V2);
LOG(ERROR) << "Can't upgrade meta from V1 to V3";
return nullptr;
} else if (version == nebula::meta::MetaVersion::V2) {
auto ret = nebula::meta::MetaVersionMan::updateMetaV2ToV3(engine);
if (!ret.ok()) {
Expand Down
2 changes: 0 additions & 2 deletions src/meta/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,6 @@ nebula_add_library(

add_dependencies(
meta_version_man_obj
meta_v1_thrift_obj
meta_v2_thrift_obj
)

Expand All @@ -122,7 +121,6 @@ set(meta_test_deps
$<TARGET_OBJECTS:meta_service_handler>
$<TARGET_OBJECTS:meta_version_man_obj>
$<TARGET_OBJECTS:meta_v2_thrift_obj>
$<TARGET_OBJECTS:meta_v1_thrift_obj>
$<TARGET_OBJECTS:meta_data_upgrade_obj>
$<TARGET_OBJECTS:storage_admin_service_handler>
$<TARGET_OBJECTS:graph_storage_service_handler>
Expand Down
277 changes: 0 additions & 277 deletions src/meta/MetaVersionMan.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
#include "meta/processors/job/JobDescription.h"
#include "meta/processors/job/JobUtils.h"
#include "meta/upgrade/MetaDataUpgrade.h"
#include "meta/upgrade/v1/MetaServiceUtilsV1.h"
#include "meta/upgrade/v2/MetaServiceUtilsV2.h"

DEFINE_bool(null_type, true, "set schema to support null type");
Expand Down Expand Up @@ -60,39 +59,6 @@ bool MetaVersionMan::setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion v
return code == nebula::cpp2::ErrorCode::SUCCEEDED;
}

// static
Status MetaVersionMan::updateMetaV1ToV2(kvstore::KVEngine* engine) {
CHECK_NOTNULL(engine);
auto snapshot = folly::sformat("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr());

std::string path = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot);
if (!fs::FileUtils::exist(path) && !fs::FileUtils::makeDir(path)) {
LOG(ERROR) << "Make checkpoint dir: " << path << " failed";
return Status::Error("Create snapshot file failed");
}

std::string dataPath = folly::sformat("{}/data", path);
auto code = engine->createCheckpoint(dataPath);
if (code != nebula::cpp2::ErrorCode::SUCCEEDED) {
LOG(ERROR) << "Create snapshot failed: " << snapshot;
return Status::Error("Create snapshot failed");
}

auto status = doUpgradeV1ToV2(engine);
if (!status.ok()) {
// rollback by snapshot
return status;
}

// delete snapshot file
auto checkpointPath = folly::sformat("{}/checkpoints/{}", engine->getDataRoot(), snapshot);

if (fs::FileUtils::exist(checkpointPath) && !fs::FileUtils::remove(checkpointPath.data(), true)) {
LOG(ERROR) << "Delete snapshot: " << snapshot << " failed, You need to delete it manually";
}
return Status::OK();
}

Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
CHECK_NOTNULL(engine);
auto snapshot = folly::format("META_UPGRADE_SNAPSHOT_{}", MetaKeyUtils::genTimestampStr()).str();
Expand Down Expand Up @@ -124,249 +90,6 @@ Status MetaVersionMan::updateMetaV2ToV3(kvstore::KVEngine* engine) {
return Status::OK();
}

// static
Status MetaVersionMan::doUpgradeV1ToV2(kvstore::KVEngine* engine) {
MetaDataUpgrade upgrader(engine);
{
// kSpacesTable
auto prefix = nebula::meta::v1::kSpacesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSpacesV1(iter->val());
}
status = upgrader.rewriteSpaces(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kPartsTable
auto prefix = nebula::meta::v1::kPartsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printParts(iter->key(), iter->val());
}
status = upgrader.rewriteParts(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kHostsTable
auto prefix = nebula::meta::v1::kHostsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printHost(iter->key(), iter->val());
}
status = upgrader.rewriteHosts(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kLeadersTable
auto prefix = nebula::meta::v1::kLeadersTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printLeaders(iter->key());
}
status = upgrader.rewriteLeaders(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kTagsTable
auto prefix = nebula::meta::v1::kTagsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSchemas(iter->val());
}
status = upgrader.rewriteSchemas(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kEdgesTable
auto prefix = nebula::meta::v1::kEdgesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printSchemas(iter->val());
}
status = upgrader.rewriteSchemas(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kIndexesTable
auto prefix = nebula::meta::v1::kIndexesTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printIndexes(iter->val());
}
status = upgrader.rewriteIndexes(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kConfigsTable
auto prefix = nebula::meta::v1::kConfigsTable;
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (FLAGS_print_info) {
upgrader.printConfigs(iter->key(), iter->val());
}
status = upgrader.rewriteConfigs(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}

{
// kJob
auto prefix = JobUtil::jobPrefix();
std::unique_ptr<kvstore::KVIterator> iter;
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (JobDescription::isJobKey(iter->key())) {
if (FLAGS_print_info) {
upgrader.printJobDesc(iter->key(), iter->val());
}
status = upgrader.rewriteJobDesc(iter->key(), iter->val());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
} else {
// the job key format is change, need to delete the old format
status = upgrader.deleteKeyVal(iter->key());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
}
iter->next();
}
}
}

// delete
{
std::vector<std::string> prefixes({nebula::meta::v1::kIndexStatusTable,
nebula::meta::v1::kDefaultTable,
nebula::meta::v1::kCurrJob,
nebula::meta::v1::kJobArchive});
std::unique_ptr<kvstore::KVIterator> iter;
for (auto& prefix : prefixes) {
auto ret = engine->prefix(prefix, &iter);
if (ret == nebula::cpp2::ErrorCode::SUCCEEDED) {
Status status = Status::OK();
while (iter->valid()) {
if (prefix == nebula::meta::v1::kIndexesTable) {
if (FLAGS_print_info) {
upgrader.printIndexes(iter->val());
}
auto oldItem = meta::v1::MetaServiceUtilsV1::parseIndex(iter->val());
auto spaceId = MetaKeyUtils::parseIndexesKeySpaceID(iter->key());
status = upgrader.deleteKeyVal(
MetaKeyUtils::indexIndexKey(spaceId, oldItem.get_index_name()));
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
}
status = upgrader.deleteKeyVal(iter->key());
if (!status.ok()) {
LOG(ERROR) << status;
return status;
}
iter->next();
}
}
}
}
if (!setMetaVersionToKV(engine, MetaVersion::V2)) {
return Status::Error("Persist meta version failed");
} else {
return Status::OK();
}
}

Status MetaVersionMan::doUpgradeV2ToV3(kvstore::KVEngine* engine) {
MetaDataUpgrade upgrader(engine);
// Step 1: Upgrade HeartBeat into machine list
Expand Down
4 changes: 0 additions & 4 deletions src/meta/MetaVersionMan.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,11 @@ class MetaVersionMan final {

static bool setMetaVersionToKV(kvstore::KVEngine* engine, MetaVersion version);

static Status updateMetaV1ToV2(kvstore::KVEngine* engine);

static Status updateMetaV2ToV3(kvstore::KVEngine* engine);

private:
static MetaVersion getVersionByHost(kvstore::KVStore* kv);

static Status doUpgradeV1ToV2(kvstore::KVEngine* engine);

static Status doUpgradeV2ToV3(kvstore::KVEngine* engine);
};

Expand Down
3 changes: 0 additions & 3 deletions src/meta/upgrade/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,12 @@
nebula_add_library(
meta_data_upgrade_obj OBJECT
MetaDataUpgrade.cpp
v1/MetaServiceUtilsV1.cpp
v2/MetaServiceUtilsV2.cpp
)

add_dependencies(
meta_data_upgrade_obj
meta_v1_thrift_obj
meta_v2_thrift_obj
)

nebula_add_subdirectory(v1)
nebula_add_subdirectory(v2)
Loading

0 comments on commit ed1f9b1

Please sign in to comment.