-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Summary: This diff adds logic to restore table schema. After this, we should be able to undo an ALTER TABLE operation! There are two important changes as part of this diff. 1) Restoring master side sys_catalog metadata. 2) Sending the restored version of the schema from the master to the TS, as part of the explicit command to restore the TS. As part of applying the restore operation on the master, we add new state tracking, which can do the diff between current sys_catalog state vs the state at the time at which we want to restore. This is done by restoring the corresponding sys_catalog snapshot into a temporary directory, with the HybridTime filter applied, for the restore_at time. We then load the relevant TABLE and TABLET data into memory and overwrite the existing rocksdb data directly in memory. This is safe to do because - It is done as part of the apply step of a raft operation, so it is already persisted and will be replayed accordingly at bootstrap, in case of a restart. - It is done on both leader and follower. Once the master state is rolled back, we then run the TS side of the restore operation. The master now sends over the restored schema information, as part of the Restore request. On the TS side, we update our tablet schema information on disk accordingly. Note: In between the master state being rolled back and all the TS processing their respective restores, there is a time window in which the master can receive heartbeats from a TS, with newer schema information than what the master has persisted. Currently, that seems to only lead to some log spew, but will be investigated later, as part of fault tolerance testing. Test Plan: ybd --gtest_filter SnapshotScheduleTest.RestoreSchema Reviewers: amitanand, bogdan Reviewed By: bogdan Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D11013
- Loading branch information
Showing
34 changed files
with
555 additions
and
101 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright (c) YugaByte, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except | ||
| // in compliance with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software distributed under the License | ||
| // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express | ||
| // or implied. See the License for the specific language governing permissions and limitations | ||
| // under the License. | ||
| // | ||
|
|
||
| #include "yb/master/restore_sys_catalog_state.h" | ||
|
|
||
| #include "yb/master/master.pb.h" | ||
| #include "yb/master/master_backup.pb.h" | ||
|
|
||
| #include "yb/util/pb_util.h" | ||
|
|
||
| namespace yb { | ||
| namespace master { | ||
|
|
||
| // Utility class to restore sys catalog. | ||
| // Initially we load tables and tablets into it, then match schedule filter. | ||
| CHECKED_STATUS RestoreSysCatalogState::LoadTable(const Slice& id, const Slice& data) { | ||
| tables_.emplace(id.ToBuffer(), VERIFY_RESULT(pb_util::ParseFromSlice<SysTablesEntryPB>(data))); | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| CHECKED_STATUS RestoreSysCatalogState::LoadTablet(const Slice& id, const Slice& data) { | ||
| tablets_.emplace( | ||
| id.ToBuffer(), VERIFY_RESULT(pb_util::ParseFromSlice<SysTabletsEntryPB>(data))); | ||
| return Status::OK(); | ||
| } | ||
|
|
||
| void AddEntry( | ||
| SysRowEntry::Type type, const std::string& id, const google::protobuf::MessageLite& pb, | ||
| SysRowEntries* out, faststring* buffer) { | ||
| auto& entry = *out->mutable_entries()->Add(); | ||
| entry.set_type(type); | ||
| entry.set_id(id); | ||
| pb_util::SerializeToString(pb, buffer); | ||
| entry.set_data(buffer->data(), buffer->size()); | ||
| } | ||
|
|
||
| Result<SysRowEntries> RestoreSysCatalogState::FilterEntries( | ||
| const SnapshotScheduleFilterPB& filter) { | ||
| SysRowEntries result; | ||
| std::unordered_set<TableId> restored_tables; | ||
| faststring buffer; | ||
| for (const auto& id_and_metadata : tables_) { | ||
| bool match; | ||
| if (id_and_metadata.second.has_index_info()) { | ||
| auto it = tables_.find(id_and_metadata.second.index_info().indexed_table_id()); | ||
| if (it == tables_.end()) { | ||
| return STATUS_FORMAT( | ||
| NotFound, "Indexed table $0 not found for index $1 ($2)", | ||
| id_and_metadata.second.index_info().indexed_table_id(), id_and_metadata.first, | ||
| id_and_metadata.second.name()); | ||
| } | ||
| match = VERIFY_RESULT(MatchTable(filter, it->first, it->second)); | ||
| } else { | ||
| match = VERIFY_RESULT(MatchTable(filter, id_and_metadata.first, id_and_metadata.second)); | ||
| } | ||
| if (!match) { | ||
| continue; | ||
| } | ||
| AddEntry(SysRowEntry::TABLE, id_and_metadata.first, id_and_metadata.second, &result, &buffer); | ||
| restored_tables.insert(id_and_metadata.first); | ||
| VLOG(2) << "Table to restore: " << id_and_metadata.first << ", " | ||
| << id_and_metadata.second.ShortDebugString(); | ||
| } | ||
| for (const auto& id_and_metadata : tablets_) { | ||
| if (restored_tables.count(id_and_metadata.second.table_id()) == 0) { | ||
| continue; | ||
| } | ||
| AddEntry(SysRowEntry::TABLET, id_and_metadata.first, id_and_metadata.second, &result, &buffer); | ||
| VLOG(2) << "Tablet to restore: " << id_and_metadata.first << ", " | ||
| << id_and_metadata.second.ShortDebugString(); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| Result<bool> RestoreSysCatalogState::MatchTable( | ||
| const SnapshotScheduleFilterPB& filter, const TableId& id, const SysTablesEntryPB& table) { | ||
| VLOG(1) << __func__ << "(" << filter.ShortDebugString() << ", " << id << ", " | ||
| << table.ShortDebugString() << ")"; | ||
| for (const auto& table_identifier : filter.tables().tables()) { | ||
| if (table_identifier.has_table_id()) { | ||
| return id == table_identifier.table_id(); | ||
| } | ||
| if (table_identifier.has_table_name()) { | ||
| return STATUS(NotSupported, "Table name filters are not implemented for PITR"); | ||
| } | ||
| return STATUS_FORMAT( | ||
| InvalidArgument, "Wrong table identifier format: $0", table_identifier); | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| } // namespace master | ||
| } // namespace yb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // Copyright (c) YugaByte, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except | ||
| // in compliance with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software distributed under the License | ||
| // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express | ||
| // or implied. See the License for the specific language governing permissions and limitations | ||
| // under the License. | ||
| // | ||
|
|
||
| #ifndef ENT_SRC_YB_MASTER_RESTORE_SYS_CATALOG_STATE_H | ||
| #define ENT_SRC_YB_MASTER_RESTORE_SYS_CATALOG_STATE_H | ||
|
|
||
| #include <unordered_map> | ||
|
|
||
| #include "yb/common/entity_ids.h" | ||
|
|
||
| #include "yb/master/master_fwd.h" | ||
| #include "yb/master/master.pb.h" | ||
|
|
||
| #include "yb/util/result.h" | ||
|
|
||
| namespace yb { | ||
| namespace master { | ||
|
|
||
| // Utility class to restore sys catalog. | ||
| // Initially we load tables and tablets into it, then match schedule filter. | ||
| class RestoreSysCatalogState { | ||
| public: | ||
| CHECKED_STATUS LoadTable(const Slice& id, const Slice& data); | ||
| CHECKED_STATUS LoadTablet(const Slice& id, const Slice& data); | ||
| Result<SysRowEntries> FilterEntries(const SnapshotScheduleFilterPB& filter); | ||
|
|
||
| private: | ||
| Result<bool> MatchTable( | ||
| const SnapshotScheduleFilterPB& filter, const TableId& id, const SysTablesEntryPB& table); | ||
|
|
||
| std::unordered_map<TableId, SysTablesEntryPB> tables_; | ||
| std::unordered_map<TabletId, SysTabletsEntryPB> tablets_; | ||
| }; | ||
|
|
||
| } // namespace master | ||
| } // namespace yb | ||
|
|
||
| #endif // ENT_SRC_YB_MASTER_RESTORE_SYS_CATALOG_STATE_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.