Skip to content

Commit

Permalink
feat(storage): another Bucket CRUD field
Browse files Browse the repository at this point in the history
This time, the `Bucket.auto_class()` gains `terminal_storage_class`, and
`OUTPUT_ONLY` `terminate_storage_class_update` fields. Only interesting
for completeness sake.
  • Loading branch information
coryan committed Oct 7, 2023
1 parent 6380988 commit 3063975
Show file tree
Hide file tree
Showing 9 changed files with 132 additions and 16 deletions.
7 changes: 6 additions & 1 deletion google/cloud/storage/bucket_autoclass.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
std::ostream& operator<<(std::ostream& os, BucketAutoclass const& rhs) {
google::cloud::internal::IosFlagsSaver flags(os);
return os << "{enabled=" << std::boolalpha << rhs.enabled << ", toggle_time="
<< google::cloud::internal::FormatRfc3339(rhs.toggle_time) << "}";
<< google::cloud::internal::FormatRfc3339(rhs.toggle_time)
<< ", terminal_storage_class=" << rhs.terminal_storage_class
<< ", terminal_storage_class_update="
<< google::cloud::internal::FormatRfc3339(
rhs.terminal_storage_class_update)
<< "}";
}

GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
Expand Down
19 changes: 17 additions & 2 deletions google/cloud/storage/bucket_autoclass.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
#include "google/cloud/storage/version.h"
#include <chrono>
#include <iosfwd>
#include <string>
#include <tuple>
#include <utility>

namespace google {
namespace cloud {
Expand All @@ -38,16 +40,29 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
*/
struct BucketAutoclass {
explicit BucketAutoclass(bool e) : enabled(e) {}
explicit BucketAutoclass(bool e, std::string tsc)
: enabled(e), terminal_storage_class(std::move(tsc)) {}
explicit BucketAutoclass(bool e, std::chrono::system_clock::time_point tp)
: enabled(e), toggle_time(tp) {}
explicit BucketAutoclass(bool e, std::chrono::system_clock::time_point tp,
std::string tsc,
std::chrono::system_clock::time_point tscu)
: enabled(e),
toggle_time(tp),
terminal_storage_class(std::move(tsc)),
terminal_storage_class_update(tscu) {}

bool enabled;
std::chrono::system_clock::time_point toggle_time;
std::string terminal_storage_class;
std::chrono::system_clock::time_point terminal_storage_class_update;
};

inline bool operator==(BucketAutoclass const& lhs, BucketAutoclass const& rhs) {
return std::tie(lhs.enabled, lhs.toggle_time) ==
std::tie(rhs.enabled, rhs.toggle_time);
return std::tie(lhs.enabled, lhs.toggle_time, lhs.terminal_storage_class,
lhs.terminal_storage_class_update) ==
std::tie(rhs.enabled, rhs.toggle_time, rhs.terminal_storage_class,
rhs.terminal_storage_class_update);
}

inline bool operator!=(BucketAutoclass const& lhs, BucketAutoclass const& rhs) {
Expand Down
7 changes: 5 additions & 2 deletions google/cloud/storage/bucket_metadata.cc
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,11 @@ BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::ResetAcl() {

BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::SetAutoclass(
BucketAutoclass const& v) {
impl_.AddSubPatch(
"autoclass", internal::PatchBuilder().SetBoolField("enabled", v.enabled));
auto builder = internal::PatchBuilder().SetBoolField("enabled", v.enabled);
if (!v.terminal_storage_class.empty()) {
builder.SetStringField("terminalStorageClass", v.terminal_storage_class);
}
impl_.AddSubPatch("autoclass", std::move(builder));
return *this;
}

Expand Down
27 changes: 24 additions & 3 deletions google/cloud/storage/bucket_metadata_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ BucketMetadata CreateBucketMetadataForTest() {
],
"autoclass": {
"enabled": true,
"toggleTime": "2022-10-07T01:02:03Z"
"toggleTime": "2022-10-07T01:02:03Z",
"terminalStorageClass": "NEARLINE",
"terminalStorageClassUpdateTime": "2022-10-07T01:02:34Z"
},
"billing": {
"requesterPays": true
Expand Down Expand Up @@ -187,8 +189,11 @@ TEST(BucketMetadataTest, Parse) {
google::cloud::internal::ParseRfc3339("2022-10-07T01:02:03Z");
ASSERT_STATUS_OK(expected_autoclass_toggle);
ASSERT_TRUE(actual.has_autoclass());
auto const expected_tscu =
google::cloud::internal::ParseRfc3339("2022-10-07T01:02:34Z");
EXPECT_EQ(actual.autoclass(),
BucketAutoclass(true, *expected_autoclass_toggle));
BucketAutoclass(true, *expected_autoclass_toggle, "NEARLINE",
*expected_tscu));

EXPECT_TRUE(actual.billing().requester_pays);
EXPECT_EQ(2, actual.cors().size());
Expand Down Expand Up @@ -318,7 +323,9 @@ TEST(BucketMetadataTest, IOStream) {
// autoclass()
EXPECT_THAT(
actual,
HasSubstr("autoclass={enabled=true, toggle_time=2022-10-07T01:02:03Z}"));
HasSubstr("autoclass={enabled=true, toggle_time=2022-10-07T01:02:03Z,"
" terminal_storage_class=NEARLINE,"
" terminal_storage_class_update=2022-10-07T01:02:34Z}"));

// billing()
EXPECT_THAT(actual, HasSubstr("enabled=true"));
Expand Down Expand Up @@ -407,6 +414,8 @@ TEST(BucketMetadataTest, ToJsonString) {
{"enabled", true},
// "toggleTime" is OUTPUT_ONLY and thus not included in the
// JSON string for create/update.
{"terminalStorageClass", "NEARLINE"},
// "terminateStorageClassUpdateTime" is OUTPUT_ONLY too.
};
EXPECT_EQ(actual["autoclass"], expected_autoclass);

Expand Down Expand Up @@ -999,6 +1008,18 @@ TEST(BucketMetadataPatchBuilder, ResetAcl) {
}

TEST(BucketMetadataPatchBuilder, SetAutoclass) {
BucketMetadataPatchBuilder builder;
builder.SetAutoclass(BucketAutoclass(true, "ARCHIVE"));

auto actual = builder.BuildPatch();
auto const json = nlohmann::json::parse(actual);
ASSERT_TRUE(json.contains("autoclass")) << json;
auto const expected_autoclass =
nlohmann::json{{"enabled", true}, {"terminalStorageClass", "ARCHIVE"}};
EXPECT_EQ(expected_autoclass, json["autoclass"]);
}

TEST(BucketMetadataPatchBuilder, SetAutoclassNoTerminal) {
BucketMetadataPatchBuilder builder;
builder.SetAutoclass(BucketAutoclass(true));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,30 @@ void SetAutoclass(google::cloud::storage::Client client,
(std::move(client), argv.at(0), enabled);
}

void SetAutoclassArchive(google::cloud::storage::Client client,
std::vector<std::string> const& argv) {
using ::google::cloud::storage::examples::Usage;
//! [set-autoclass-archive]
namespace gcs = ::google::cloud::storage;
[](gcs::Client client, std::string const& bucket_name) {
auto metadata = client.PatchBucket(
bucket_name,
gcs::BucketMetadataPatchBuilder().SetAutoclass(
gcs::BucketAutoclass{true, gcs::storage_class::Archive()}));
if (!metadata) throw google::cloud::Status(std::move(metadata).status());

std::cout << "The autoclass configuration for bucket " << bucket_name
<< " was successfully updated.";
if (!metadata->has_autoclass()) {
std::cout << " The bucket no longer has an autoclass configuration.\n";
return;
}
std::cout << " The new configuration is " << metadata->autoclass() << "\n";
}
//! [set-autoclass-archive]
(std::move(client), argv.at(0));
}

void RunAll(std::vector<std::string> const& argv) {
namespace examples = ::google::cloud::storage::examples;
namespace gcs = ::google::cloud::storage;
Expand All @@ -83,15 +107,19 @@ void RunAll(std::vector<std::string> const& argv) {
auto const project_id =
google::cloud::internal::GetEnv("GOOGLE_CLOUD_PROJECT").value();
auto generator = google::cloud::internal::DefaultPRNG(std::random_device{}());
// We need multiple buckets because in production the autoclass state cannot
// be reset until 24 hours have elapsed.
auto const bucket_name_enabled = examples::MakeRandomBucketName(generator);
auto const bucket_name_disabled = examples::MakeRandomBucketName(generator);
auto const bucket_name_archive = examples::MakeRandomBucketName(generator);
auto const object_name =
examples::MakeRandomObjectName(generator, "object-") + ".txt";
auto client = gcs::Client();

std::cout << "\nCreating buckets to run the example:"
<< "\nEnabled Autoclass: " << bucket_name_enabled
<< "\nDisabled Autoclass: " << bucket_name_disabled << std::endl;
<< "\nDisabled Autoclass: " << bucket_name_disabled
<< "\nArchive Autclass: " << bucket_name_archive << std::endl;
// In GCS a single project cannot create or delete buckets more often than
// once every two seconds. We will pause until that time before deleting the
// bucket.
Expand All @@ -109,6 +137,13 @@ void RunAll(std::vector<std::string> const& argv) {
gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{false}),
examples::CreateBucketOptions())
.value();
if (!examples::UsingEmulator()) std::this_thread::sleep_for(kBucketPeriod);
(void)client
.CreateBucketForProject(
bucket_name_archive, project_id,
gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{false}),
examples::CreateBucketOptions())
.value();
auto const pause = std::chrono::steady_clock::now() + kBucketPeriod;

std::cout << "\nRunning GetAutoclass() example [enabled]" << std::endl;
Expand All @@ -120,10 +155,15 @@ void RunAll(std::vector<std::string> const& argv) {
std::cout << "\nRunning SetAutoclass() example" << std::endl;
SetAutoclass(client, {bucket_name_enabled, "false"});

std::cout << "\nRunning SetAutoclassArchive() example" << std::endl;
SetAutoclassArchive(client, {bucket_name_archive});

if (!examples::UsingEmulator()) std::this_thread::sleep_until(pause);
(void)examples::RemoveBucketAndContents(client, bucket_name_enabled);
if (!examples::UsingEmulator()) std::this_thread::sleep_for(kBucketPeriod);
(void)examples::RemoveBucketAndContents(client, bucket_name_disabled);
if (!examples::UsingEmulator()) std::this_thread::sleep_for(kBucketPeriod);
(void)examples::RemoveBucketAndContents(client, bucket_name_archive);
}

} // namespace
Expand All @@ -135,6 +175,8 @@ int main(int argc, char* argv[]) {
GetAutoclass),
examples::CreateCommandEntry(
"set-autoclass", {"<bucket-name>", "<enabled>"}, SetAutoclass),
examples::CreateCommandEntry("set-autoclass-archive", {"<bucket-name>"},
SetAutoclassArchive),
{"auto", RunAll},
});
return example.Run(argc, argv);
Expand Down
14 changes: 10 additions & 4 deletions google/cloud/storage/internal/bucket_metadata_parser.cc
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,11 @@ Status ParseAutoclass(BucketMetadata& meta, nlohmann::json const& json) {
if (!enabled) return std::move(enabled).status();
auto toggle = internal::ParseTimestampField(*f, "toggleTime");
if (!toggle) return std::move(toggle).status();
meta.set_autoclass(BucketAutoclass{*enabled, *toggle});
auto tsc = f->value("terminalStorageClass", "");
auto tscu =
internal::ParseTimestampField(*f, "terminalStorageClassUpdateTime");
if (!tscu) return std::move(tscu).status();
meta.set_autoclass(BucketAutoclass{*enabled, *toggle, std::move(tsc), *tscu});
return Status{};
}

Expand Down Expand Up @@ -300,9 +304,11 @@ void ToJsonCors(nlohmann::json& json, BucketMetadata const& meta) {

void ToJsonAutoclass(nlohmann::json& json, BucketMetadata const& meta) {
if (!meta.has_autoclass()) return;
json["autoclass"] = nlohmann::json{
{"enabled", meta.autoclass().enabled},
};
auto a = nlohmann::json{{"enabled", meta.autoclass().enabled}};
if (!meta.autoclass().terminal_storage_class.empty()) {
a["terminalStorageClass"] = meta.autoclass().terminal_storage_class;
}
json["autoclass"] = std::move(a);
}

void ToJsonBilling(nlohmann::json& json, BucketMetadata const& meta) {
Expand Down
10 changes: 10 additions & 0 deletions google/cloud/storage/internal/grpc/bucket_metadata_parser.cc
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,10 @@ google::storage::v2::Bucket::Autoclass ToProto(
result.set_enabled(rhs.enabled);
*result.mutable_toggle_time() =
google::cloud::internal::ToProtoTimestamp(rhs.toggle_time);
result.set_terminal_storage_class(rhs.terminal_storage_class);
*result.mutable_terminal_storage_class_update_time() =
google::cloud::internal::ToProtoTimestamp(
rhs.terminal_storage_class_update);
return result;
}

Expand All @@ -209,6 +213,12 @@ storage::BucketAutoclass FromProto(
result.toggle_time =
google::cloud::internal::ToChronoTimePoint(rhs.toggle_time());
}
result.terminal_storage_class = rhs.terminal_storage_class();
if (rhs.has_terminal_storage_class_update_time()) {
result.terminal_storage_class_update =
google::cloud::internal::ToChronoTimePoint(
rhs.terminal_storage_class_update_time());
}
return result;
}

Expand Down
16 changes: 14 additions & 2 deletions google/cloud/storage/internal/grpc/bucket_metadata_parser_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,11 @@ TEST(GrpcBucketMetadataParser, BucketAllFieldsRoundtrip) {
autoclass {
enabled: true
toggle_time { seconds: 1665108184 nanos: 123456000 }
terminal_storage_class: "NEARLINE"
terminal_storage_class_update_time: {
seconds: 1665108124
nanos: 123456000
}
}
)pb";
EXPECT_TRUE(google::protobuf::TextFormat::ParseFromString(kText, &input));
Expand Down Expand Up @@ -234,7 +239,9 @@ TEST(GrpcBucketMetadataParser, BucketAllFieldsRoundtrip) {
"selfLink": "https://www.googleapis.com/storage/v1/b/test-bucket-id",
"autoclass": {
"enabled": true,
"toggleTime": "2022-10-07T02:03:04.123456000Z"
"toggleTime": "2022-10-07T02:03:04.123456000Z",
"terminalStorageClass": "NEARLINE",
"terminalStorageClassUpdateTime": "2022-10-07T02:02:04.123456000Z"
}
})""");
ASSERT_THAT(expected, IsOk());
Expand All @@ -254,13 +261,18 @@ TEST(GrpcBucketMetadataParser, BucketAutoclassRoundtrip) {
auto constexpr kText = R"pb(
enabled: true
toggle_time { seconds: 1665108184 nanos: 123456000 }
terminal_storage_class: "NEARLINE"
terminal_storage_class_update_time { seconds: 1665108194 nanos: 123456000 }
)pb";
google::storage::v2::Bucket::Autoclass start;
EXPECT_TRUE(google::protobuf::TextFormat::ParseFromString(kText, &start));
auto const expected_toggle =
google::cloud::internal::ParseRfc3339("2022-10-07T02:03:04.123456000Z");
ASSERT_STATUS_OK(expected_toggle);
auto const expected = storage::BucketAutoclass{true, *expected_toggle};
auto const expected_tscu =
google::cloud::internal::ParseRfc3339("2022-10-07T02:03:14.123456000Z");
auto const expected = storage::BucketAutoclass{true, *expected_toggle,
"NEARLINE", *expected_tscu};
auto const middle = FromProto(start);
EXPECT_EQ(middle, expected);
auto const end = ToProto(middle);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ TEST(GrpcBucketRequestParser, CreateBucketMetadataAllOptions) {
autoclass {
enabled: true
toggle_time {}
terminal_storage_class: "NEARLINE"
terminal_storage_class_update_time {}
}
billing { requester_pays: true }
default_object_acl {
Expand Down Expand Up @@ -137,7 +139,7 @@ TEST(GrpcBucketRequestParser, CreateBucketMetadataAllOptions) {
.set_name("test-bucket")
.set_storage_class("NEARLINE")
.set_location("us-central1")
.set_autoclass(storage::BucketAutoclass{true})
.set_autoclass(storage::BucketAutoclass{true, "NEARLINE"})
.set_billing(storage::BucketBilling(true))
.set_rpo(storage::RpoAsyncTurbo())
.set_versioning(storage::BucketVersioning(true))
Expand Down

0 comments on commit 3063975

Please sign in to comment.