-
Notifications
You must be signed in to change notification settings - Fork 9
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
New Functionality to support a Key Prefix and EC2InstanceProfile IAM #9
base: master
Are you sure you want to change the base?
Changes from all commits
b8e89ba
8bcaeab
033f241
fcc8d27
96ac109
53c3720
eaf6c98
ad78eb6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,3 +2,4 @@ bin/ | |
obj/ | ||
/packages | ||
/.nuget/NuGet.exe | ||
/.vs |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -26,25 +26,48 @@ public class AmazonBlobStorageProvider : CloudBlobStorageProvider | |
/// <param name="config">The collection of parameters (each by its name and value) of the current provider's configuration settings.</param> | ||
protected override void InitializeStorage(NameValueCollection config) | ||
{ | ||
var useIamInstanceRoleValue = "false"; | ||
if (config.AllKeys.Contains(UseIamInstanceRoleKey)) | ||
useIamInstanceRoleValue = config[UseIamInstanceRoleKey].Trim(); | ||
|
||
if (!bool.TryParse(useIamInstanceRoleValue, out this.useIamInstanceRole)) | ||
throw new ConfigurationException("'{0}' unable to parse {1}".Arrange(UseIamInstanceRoleKey, useIamInstanceRoleValue)); | ||
|
||
this.accessKeyId = config[AccessKeyIdKey].Trim(); | ||
if (String.IsNullOrEmpty(this.accessKeyId)) | ||
if (!this.useIamInstanceRole && String.IsNullOrEmpty(this.accessKeyId)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Change to AccessKeyId and SecretKeyId parameters to only validate if we are not using the EC2 Instance Profile Role |
||
throw new ConfigurationException("'{0}' is required.".Arrange(AccessKeyIdKey)); | ||
|
||
this.secretKey = config[SecretKeyKey].Trim(); | ||
if (String.IsNullOrEmpty(this.secretKey)) | ||
if (!this.useIamInstanceRole && String.IsNullOrEmpty(this.secretKey)) | ||
throw new ConfigurationException("'{0}' is required.".Arrange(SecretKeyKey)); | ||
|
||
this.bucketName = config[BucketNameKey].Trim(); | ||
if (String.IsNullOrEmpty(this.bucketName)) | ||
throw new ConfigurationException("'{0}' is required.".Arrange(BucketNameKey)); | ||
|
||
if (config.AllKeys.Contains(KeyPrefixKey)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Optional configuration item to allow the use of a key prefix on files uploaded to the S3 Bucket |
||
this.keyPrefix = config[KeyPrefixKey].Trim(); | ||
|
||
if (string.IsNullOrWhiteSpace(this.keyPrefix)) | ||
this.keyPrefix = ""; | ||
|
||
if (config.AllKeys.Contains(SchemeKey)) | ||
this.scheme = config[SchemeKey].Trim(); | ||
|
||
if (string.IsNullOrWhiteSpace(this.scheme)) | ||
this.scheme = "https"; | ||
|
||
string regionEndpointString = config[RegionEndpointKey].Trim(); | ||
var endpointField = typeof(RegionEndpoint).GetField(regionEndpointString, BindingFlags.Static | BindingFlags.Public); | ||
if ((string.IsNullOrWhiteSpace(regionEndpointString)) || (endpointField == null)) | ||
throw new ConfigurationException("'{0}' is required.".Arrange(RegionEndpointKey)); | ||
|
||
var regionEndpoint = (RegionEndpoint)endpointField.GetValue(null); | ||
this.transferUtility = new TransferUtility(accessKeyId, secretKey, regionEndpoint); | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we're using the EC2 Instance Profile Role, don't pass accesskey and secretaccesskey into the TransferUtility constructor, the AWS SDK will use the instance's role by default There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @BenWolstencroft, Exception 1 of 4: We need proper documentation, how exactly this setup works, and probably cover this scenario described above. |
||
if (this.useIamInstanceRole) | ||
this.transferUtility = new TransferUtility(regionEndpoint); | ||
else | ||
this.transferUtility = new TransferUtility(accessKeyId, secretKey, regionEndpoint); | ||
} | ||
|
||
/// <summary> | ||
|
@@ -54,7 +77,7 @@ protected override void InitializeStorage(NameValueCollection config) | |
/// <returns>The resolved content item's external URL on the remote blob storage.</returns> | ||
public override string GetItemUrl(IBlobContentLocation content) | ||
{ | ||
return string.Concat("http://", this.bucketName, ".s3.amazonaws.com/", content.FilePath); | ||
return string.Concat(this.scheme, "://", this.bucketName, ".s3.amazonaws.com/", "{0}{1}".Arrange(this.keyPrefix, content.FilePath)); | ||
} | ||
|
||
/// <summary> | ||
|
@@ -67,9 +90,9 @@ public override void Copy(IBlobContentLocation source, IBlobContentLocation dest | |
var request = new CopyObjectRequest() | ||
{ | ||
SourceBucket = this.bucketName, | ||
SourceKey = source.FilePath, | ||
SourceKey = "{0}{1}".Arrange(this.keyPrefix, source.FilePath), | ||
DestinationBucket = this.bucketName, | ||
DestinationKey = destination.FilePath, | ||
DestinationKey = "{0}{1}".Arrange(this.keyPrefix, destination.FilePath), | ||
CannedACL = S3CannedACL.PublicRead | ||
}; | ||
|
||
|
@@ -84,18 +107,19 @@ public override void Copy(IBlobContentLocation source, IBlobContentLocation dest | |
public override void SetProperties(IBlobContentLocation location, IBlobProperties properties) | ||
{ | ||
//No properties to set by default | ||
var req = new CopyObjectRequest() { | ||
var req = new CopyObjectRequest() | ||
{ | ||
MetadataDirective = S3MetadataDirective.REPLACE, | ||
SourceBucket = this.bucketName, | ||
SourceKey = location.FilePath, | ||
SourceKey = "{0}{1}".Arrange(this.keyPrefix, location.FilePath), | ||
DestinationBucket = this.bucketName, | ||
DestinationKey = location.FilePath, | ||
DestinationKey = "{0}{1}".Arrange(this.keyPrefix, location.FilePath), | ||
CannedACL = S3CannedACL.PublicRead | ||
}; | ||
|
||
req.Headers.CacheControl = properties.CacheControl; | ||
req.Headers.ContentType = properties.ContentType; | ||
|
||
transferUtility.S3Client.CopyObject(req); | ||
} | ||
|
||
|
@@ -108,8 +132,8 @@ public override IBlobProperties GetProperties(IBlobContentLocation location) | |
{ | ||
var request = new GetObjectRequest() | ||
{ | ||
BucketName = this.bucketName, | ||
Key = location.FilePath | ||
BucketName = this.bucketName, | ||
Key = "{0}{1}".Arrange(this.keyPrefix, location.FilePath) | ||
}; | ||
GetObjectResponse response = transferUtility.S3Client.GetObject(request); | ||
|
||
|
@@ -132,19 +156,19 @@ public override long Upload(IBlobContent content, Stream source, int bufferSize) | |
var request = new TransferUtilityUploadRequest() | ||
{ | ||
BucketName = this.bucketName, | ||
Key = content.FilePath, | ||
Key = "{0}{1}".Arrange(this.keyPrefix, content.FilePath), | ||
PartSize = bufferSize, | ||
ContentType = content.MimeType, | ||
CannedACL = S3CannedACL.PublicRead | ||
}; | ||
|
||
//get it before the upload, because afterwards the stream is closed already | ||
long sourceLength = source.Length; | ||
using (MemoryStream str = new MemoryStream()) | ||
{ | ||
source.CopyTo(str); | ||
request.InputStream = str; | ||
|
||
this.transferUtility.Upload(request); | ||
} | ||
return sourceLength; | ||
|
@@ -169,8 +193,8 @@ public override Stream GetDownloadStream(IBlobContent content) | |
{ | ||
TransferUtilityOpenStreamRequest request = new TransferUtilityOpenStreamRequest() | ||
{ | ||
BucketName = this.bucketName, | ||
Key = content.FilePath | ||
BucketName = this.bucketName, | ||
Key = "{0}{1}".Arrange(this.keyPrefix, content.FilePath) | ||
}; | ||
var stream = this.transferUtility.OpenStream(request); | ||
return stream; | ||
|
@@ -185,7 +209,7 @@ public override void Delete(IBlobContentLocation location) | |
var request = new DeleteObjectRequest() | ||
{ | ||
BucketName = this.bucketName, | ||
Key = location.FilePath | ||
Key = "{0}{1}".Arrange(this.keyPrefix, location.FilePath) | ||
}; | ||
transferUtility.S3Client.DeleteObject(request); | ||
} | ||
|
@@ -200,15 +224,15 @@ public override bool BlobExists(IBlobContentLocation location) | |
var request = new GetObjectRequest() | ||
{ | ||
BucketName = this.bucketName, | ||
Key = location.FilePath | ||
Key = "{0}{1}".Arrange(this.keyPrefix, location.FilePath) | ||
}; | ||
try | ||
{ | ||
var response = transferUtility.S3Client.GetObject(request); | ||
return true; | ||
} | ||
catch (AmazonS3Exception err) | ||
{ | ||
{ | ||
} | ||
return false; | ||
} | ||
|
@@ -217,18 +241,25 @@ public override bool BlobExists(IBlobContentLocation location) | |
|
||
#region Properties | ||
|
||
public const string UseIamInstanceRoleKey = "useIamInstanceRole"; | ||
public const string AccessKeyIdKey = "accessKeyId"; | ||
public const string SecretKeyKey = "secretKey"; | ||
public const string BucketNameKey = "bucketName"; | ||
public const string KeyPrefixKey = "keyPrefix"; | ||
public const string RegionEndpointKey = "regionEndpoint"; | ||
public const string SchemeKey = "scheme"; | ||
|
||
#endregion | ||
|
||
#region Fields | ||
|
||
private bool useIamInstanceRole = false; | ||
private string accessKeyId = ""; | ||
private string secretKey = ""; | ||
private string bucketName = ""; | ||
private string keyPrefix = ""; | ||
private string scheme = ""; | ||
|
||
TransferUtility transferUtility; | ||
|
||
#endregion | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Optional configuration item to allow the use of the EC2 Instance Profile Role, defaults to false, maintaining backwards compatibility