Skip to content

Commit

Permalink
Add ability to set metadata for object uploads in transfer library.
Browse files Browse the repository at this point in the history
  • Loading branch information
@kahkeng
kahkeng committed Jan 13, 2016
1 parent eab46cc commit 23b358b
Show file tree
Hide file tree
Showing 5 changed files with 176 additions and 4 deletions.
96 changes: 96 additions & 0 deletions aws-cpp-sdk-transfer-tests/TransferTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1188,5 +1188,101 @@ TEST_F(TransferTests, ScopeTests)
}
}

// Single part upload with metadata specified
TEST_F(TransferTests, SinglePartUploadWithMetadataTest)
{
if (EmptyBucket(GetTestBucketName()))
{
WaitForBucketToEmpty(GetTestBucketName());
}
GetObjectRequest getObjectRequest;
getObjectRequest.SetBucket(GetTestBucketName());
getObjectRequest.SetKey(TEST_FILE_NAME);

GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
EXPECT_FALSE(getObjectOutcome.IsSuccess());

Aws::Map<Aws::String, Aws::String> metadata;
metadata["key1"] = "val1";
metadata["key2"] = "val2";
const bool cCreateBucket = false;
const bool cConsistencyChecks = false;
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(TEST_FILE_NAME, GetTestBucketName(), "", "", metadata, cCreateBucket, cConsistencyChecks);

ASSERT_FALSE(requestPtr->IsDone());

WaitForUploadAndUpdate(requestPtr, 100.0f);

ASSERT_TRUE(requestPtr->IsDone());

ASSERT_TRUE(requestPtr->CompletedSuccessfully());

WaitForObjectToPropagate(GetTestBucketName(), TEST_FILE_NAME);

// Check the metadata matches
HeadObjectRequest headObjectRequest;
headObjectRequest.SetBucket(GetTestBucketName());
headObjectRequest.SetKey(TEST_FILE_NAME);

HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
ASSERT_TRUE(headObjectOutcome.IsSuccess());

Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
ASSERT_EQ(2u, headObjectMetadata.size());
ASSERT_EQ("val1", headObjectMetadata["key1"]);
ASSERT_EQ("val2", headObjectMetadata["key2"]);

}

// Multipart upload with metadata specified
TEST_F(TransferTests, MultipartUploadWithMetadataTest)
{
if (EmptyBucket(GetTestBucketName()))
{
WaitForBucketToEmpty(GetTestBucketName());
}

GetObjectRequest getObjectRequest;
getObjectRequest.SetBucket(GetTestBucketName());
getObjectRequest.SetKey(MEDIUM_FILE_KEY);

GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
EXPECT_FALSE(getObjectOutcome.IsSuccess());

ListMultipartUploadsRequest listMultipartRequest;

listMultipartRequest.SetBucket(GetTestBucketName());

Aws::Map<Aws::String, Aws::String> metadata;
metadata["key1"] = "val1";
metadata["key2"] = "val2";
const bool cCreateBucket = false;
const bool cConsistencyChecks = false;
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(MEDIUM_TEST_FILE_NAME, GetTestBucketName(), MEDIUM_FILE_KEY, "", metadata, cCreateBucket, cConsistencyChecks);

ASSERT_FALSE(requestPtr->IsDone());

WaitForUploadAndUpdate(requestPtr, 100.0f);

ASSERT_TRUE(requestPtr->IsDone());
ASSERT_TRUE(requestPtr->CompletedSuccessfully());

WaitForObjectToPropagate(GetTestBucketName(), MEDIUM_FILE_KEY);

// Check the metadata matches
HeadObjectRequest headObjectRequest;
headObjectRequest.SetBucket(GetTestBucketName());
headObjectRequest.SetKey(MEDIUM_FILE_KEY);

HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
ASSERT_TRUE(headObjectOutcome.IsSuccess());

Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
ASSERT_EQ(2u, headObjectMetadata.size());
ASSERT_EQ("val1", headObjectMetadata["key1"]);
ASSERT_EQ("val2", headObjectMetadata["key2"]);

}


}
9 changes: 6 additions & 3 deletions aws-cpp-sdk-transfer/include/aws/transfer/TransferClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,13 @@ class AWS_TRANSFER_API TransferClient
TransferClient(const std::shared_ptr<Aws::S3::S3Client>& s3Client, const TransferClientConfiguration& config);
~TransferClient();

// Single entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
// Entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
// unnecessarily as the request waits for S3 to propagate the bucket
// All queries about the upload after this point can be found in UploadFileRequest's interface
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, bool createBucket = false, bool doConsistencyChecks = false);
// Entry point similar to above but with metadata specified
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket = false, bool doConsistencyChecks = false);
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket = false, bool doConsistencyChecks = false);

// User requested upload cancels should go through here
void CancelUpload(std::shared_ptr<UploadFileRequest>& fileRequest) const;
Expand All @@ -84,7 +87,7 @@ class AWS_TRANSFER_API TransferClient
private:

void UploadFileInternal(std::shared_ptr<UploadFileRequest>& fileRequest);

void ProcessSingleBuffer(std::shared_ptr<UploadFileRequest>& request, const std::shared_ptr<UploadBuffer>& buffer);

void CancelUploadInternal(std::shared_ptr<UploadFileRequest>& fileRequest) const;
Expand Down Expand Up @@ -168,4 +171,4 @@ class AWS_TRANSFER_API TransferClient
} // namespace Transfer
} // namespace Aws

#endif
#endif
18 changes: 18 additions & 0 deletions aws-cpp-sdk-transfer/include/aws/transfer/UploadFileRequest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
bool createBucket,
bool doConsistencyChecks);
UploadFileRequest(const Aws::String& fileName,
const Aws::String& bucketName,
const Aws::String& keyName,
const Aws::String& contentType,
const Aws::Map<Aws::String, Aws::String>& metadata,
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
bool createBucket,
bool doConsistencyChecks);
UploadFileRequest(const Aws::String& fileName,
const Aws::String& bucketName,
const Aws::String& keyName,
const Aws::String& contentType,
Aws::Map<Aws::String, Aws::String>&& metadata,
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
bool createBucket,
bool doConsistencyChecks);
~UploadFileRequest();

// How many parts have we at least begun to upload
Expand Down Expand Up @@ -255,6 +271,8 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
Aws::List<std::shared_ptr<UploadBuffer> > m_buffersReady;

Aws::String m_contentType;
Aws::Map<Aws::String, Aws::String> m_metadata;
bool m_metadataHasBeenSet;
Aws::String m_uploadId;
uint32_t m_createMultipartRetries;
uint32_t m_createBucketRetries;
Expand Down
18 changes: 18 additions & 0 deletions aws-cpp-sdk-transfer/source/transfer/TransferClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,24 @@ std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String&
return request;
}

std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket, bool doConsistencyChecks)
{
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);

UploadFileInternal(request);

return request;
}

std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket, bool doConsistencyChecks)
{
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);

UploadFileInternal(request);

return request;
}

void TransferClient::UploadFileInternal(std::shared_ptr<UploadFileRequest>& request)
{

Expand Down
39 changes: 38 additions & 1 deletion aws-cpp-sdk-transfer/source/transfer/UploadFileRequest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ m_bucketPropagated(false),
m_totalParts(0),
m_fileStream(fileName.c_str(), std::ios::binary | std::ios::ate),
m_contentType(contentType),
m_metadataHasBeenSet(false),
m_createMultipartRetries(0),
m_createBucketRetries(0),
m_completeRetries(0),
Expand Down Expand Up @@ -105,6 +106,34 @@ m_headBucketRetries(0)
}
}

UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
const Aws::String& bucketName,
const Aws::String& keyName,
const Aws::String& contentType,
const Aws::Map<Aws::String, Aws::String>& metadata,
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
bool createBucket,
bool doConsistencyChecks) :
UploadFileRequest(fileName, bucketName, keyName, contentType, s3Client, createBucket, doConsistencyChecks)
{
m_metadataHasBeenSet = true;
m_metadata = metadata;
}

UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
const Aws::String& bucketName,
const Aws::String& keyName,
const Aws::String& contentType,
Aws::Map<Aws::String, Aws::String>&& metadata,
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
bool createBucket,
bool doConsistencyChecks) :
UploadFileRequest(fileName, bucketName, keyName, contentType, s3Client, createBucket, doConsistencyChecks)
{
m_metadataHasBeenSet = true;
m_metadata = metadata;
}

UploadFileRequest::~UploadFileRequest()
{
m_fileStream.close();
Expand Down Expand Up @@ -202,6 +231,10 @@ bool UploadFileRequest::CreateMultipartUpload()
{
createMultipartUploadRequest.SetContentType(m_contentType);
}
if (m_metadataHasBeenSet)
{
createMultipartUploadRequest.SetMetadata(m_metadata);
}

std::shared_ptr<Aws::Client::AsyncCallerContext> context = Aws::MakeShared<UploadFileContext>(ALLOCATION_TAG, shared_from_this());

Expand Down Expand Up @@ -507,7 +540,7 @@ bool UploadFileRequest::ProcessBuffer(const std::shared_ptr<UploadBuffer>& buffe
return false;
}

if (GetTotalParts() == 1)
if (IsSinglePartUpload())
{
// Don't need more than one part, do everything now
return DoSingleObjectUpload(streamBuf, bytesRead);
Expand Down Expand Up @@ -774,6 +807,10 @@ bool UploadFileRequest::DoSingleObjectUpload(std::shared_ptr<Aws::IOStream>& str
{
putObjectRequest.SetContentType(m_contentType);
}
if (m_metadataHasBeenSet)
{
putObjectRequest.SetMetadata(m_metadata);
}
putObjectRequest.SetKey(GetKeyName());

putObjectRequest.SetDataSentEventHandler(std::bind(&UploadFileRequest::OnDataSent, this, std::placeholders::_1, std::placeholders::_2));
Expand Down

0 comments on commit 23b358b

Please sign in to comment.