Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

enable different endpoint to S3 bitstorage #338

Merged
merged 2 commits into from
Jun 22, 2023
Merged

enable different endpoint to S3 bitstorage #338

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c870b2
enable different endpoint to S3 bitstorage
MajoBerger Jun 22, 2023
4dd4ac3
modified tests
MajoBerger Jun 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 50 additions & 4 deletions dspace-api/src/main/java/org/dspace/storage/bitstore/S3BitStoreService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
Expand Down Expand Up @@ -88,12 +89,14 @@ public class S3BitStoreService extends BaseBitStoreService {
protected static final int directoryLevels = 3;

private boolean enabled = false;

private String awsAccessKey;
private String awsSecretKey;
private String awsRegionName;
private boolean useRelativePath;

private String endpoint;
private boolean pathStyleAccessEnabled;

/**
* container for all the assets
*/
Expand All @@ -119,7 +122,7 @@ public class S3BitStoreService extends BaseBitStoreService {
= DSpaceServicesFactory.getInstance().getConfigurationService();

/**
* Utility method for generate AmazonS3 builder
* Utility method for generate AmazonS3 builder with specific region
*
* @param regions wanted regions in client
* @param awsCredentials credentials of the client
Expand All @@ -135,6 +138,24 @@ protected static Supplier<AmazonS3> amazonClientBuilderBy(
.build();
}

/**
* Utility method for generate AmazonS3 builder with specific endpoint
*
* @param endpointConfiguration configuration of endpoint
* @param awsCredentials credentials of the client
* @param pathStyleAccessEnabled enable path style access to S3 service
* @return builder with the specified parameters
*/
protected static Supplier<AmazonS3> amazonClientBuilderBy(
@NotNull AwsClientBuilder.EndpointConfiguration endpointConfiguration,
@NotNull AWSCredentials awsCredentials,
@NotNull boolean pathStyleAccessEnabled
) {
return () -> AmazonS3ClientBuilder.standard()
.withPathStyleAccessEnabled( pathStyleAccessEnabled)
.withEndpointConfiguration(endpointConfiguration)
.withCredentials(new AWSStaticCredentialsProvider(awsCredentials)).build();
}
public S3BitStoreService() {}

/**
Expand Down Expand Up @@ -168,7 +189,16 @@ public void init() throws IOException {
}

try {
if (StringUtils.isNotBlank(getAwsAccessKey()) && StringUtils.isNotBlank(getAwsSecretKey())) {
if (StringUtils.isNotBlank(getEndpoint())) {
log.info("Creating s3service from different endpoint than amazon: " + getEndpoint());
BasicAWSCredentials credentials = new BasicAWSCredentials(getAwsAccessKey(), getAwsSecretKey());
AwsClientBuilder.EndpointConfiguration ec =
new AwsClientBuilder.EndpointConfiguration(getEndpoint(), "");
s3Service = FunctionalUtils.getDefaultOrBuild(
this.s3Service,
amazonClientBuilderBy(ec, credentials, getPathStyleAccessEnabled())
);
} else if (StringUtils.isNotBlank(getAwsAccessKey()) && StringUtils.isNotBlank(getAwsSecretKey())) {
log.warn("Use local defined S3 credentials");
// region
Regions regions = Regions.DEFAULT_REGION;
Expand Down Expand Up @@ -205,7 +235,7 @@ public void init() throws IOException {
}

try {
if (!s3Service.doesBucketExist(bucketName)) {
if (!s3Service.doesBucketExistV2(bucketName)) {
s3Service.createBucket(bucketName);
log.info("Creating new S3 Bucket: " + bucketName);
}
Expand Down Expand Up @@ -448,6 +478,22 @@ public void setAwsAccessKey(String awsAccessKey) {
this.awsAccessKey = awsAccessKey;
}

@Autowired(required = true)
public void setPathStyleAccessEnabled(boolean pathStyleAccessEnabled) {
this.pathStyleAccessEnabled = pathStyleAccessEnabled;
}

public boolean getPathStyleAccessEnabled() {
return this.pathStyleAccessEnabled;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}

public String getEndpoint() {
return this.endpoint;
}

public String getAwsSecretKey() {
return awsSecretKey;
}
Expand Down
10 changes: 5 additions & 5 deletions dspace-api/src/test/java/org/dspace/storage/bitstore/S3BitStoreServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public class S3BitStoreServiceTest extends AbstractUnitTest {

@Before
public void setUp() throws Exception {
this.s3BitStoreService = new S3BitStoreService(s3Service, tm);
this.s3BitStoreService = new S3BitStoreService(s3Service, tm);
}

private Supplier<AmazonS3> mockedServiceSupplier() {
Expand All @@ -82,13 +82,13 @@ private Supplier<AmazonS3> mockedServiceSupplier() {
public void givenBucketWhenInitThenUsesSameBucket() throws IOException {
String bucketName = "Bucket0";
s3BitStoreService.setBucketName(bucketName);
when(this.s3Service.doesBucketExist(bucketName)).thenReturn(false);
when(this.s3Service.doesBucketExistV2(bucketName)).thenReturn(false);

assertThat(s3BitStoreService.getAwsRegionName(), isEmptyOrNullString());

this.s3BitStoreService.init();

verify(this.s3Service).doesBucketExist(bucketName);
verify(this.s3Service).doesBucketExistV2(bucketName);
verify(this.s3Service, Mockito.times(1)).createBucket(bucketName);
assertThat(s3BitStoreService.getAwsAccessKey(), isEmptyOrNullString());
assertThat(s3BitStoreService.getAwsSecretKey(), isEmptyOrNullString());
Expand All @@ -98,7 +98,7 @@ public void givenBucketWhenInitThenUsesSameBucket() throws IOException {
@Test
public void givenEmptyBucketWhenInitThenUsesDefaultBucket() throws IOException {
assertThat(s3BitStoreService.getBucketName(), isEmptyOrNullString());
when(this.s3Service.doesBucketExist(startsWith(S3BitStoreService.DEFAULT_BUCKET_PREFIX))).thenReturn(false);
when(this.s3Service.doesBucketExistV2(startsWith(S3BitStoreService.DEFAULT_BUCKET_PREFIX))).thenReturn(false);
assertThat(s3BitStoreService.getAwsRegionName(), isEmptyOrNullString());

this.s3BitStoreService.init();
Expand All @@ -116,7 +116,7 @@ public void givenAccessKeysWhenInitThenVerifiesCorrectBuilderCreation() throws I
assertThat(s3BitStoreService.getAwsSecretKey(), isEmptyOrNullString());
assertThat(s3BitStoreService.getBucketName(), isEmptyOrNullString());
assertThat(s3BitStoreService.getAwsRegionName(), isEmptyOrNullString());
when(this.s3Service.doesBucketExist(startsWith(S3BitStoreService.DEFAULT_BUCKET_PREFIX))).thenReturn(false);
when(this.s3Service.doesBucketExistV2(startsWith(S3BitStoreService.DEFAULT_BUCKET_PREFIX))).thenReturn(false);

final String awsAccessKey = "ACCESS_KEY";
final String awsSecretKey = "SECRET_KEY";
Expand Down
7 changes: 6 additions & 1 deletion dspace/config/modules/assetstore.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,9 @@ assetstore.s3.awsSecretKey =

# If the credentials are left empty,
# then this setting is ignored and the default AWS region will be used.
assetstore.s3.awsRegionName =
assetstore.s3.awsRegionName =

# Configuring S3 with different endpoint than amazon can require pathstyle access which can be configured here
assetstore.s3.pathStyleAccessEnabled = false
# Leave empty to use default (Amazon AWS) endpoint
assetstore.s3.endpoint =
4 changes: 4 additions & 0 deletions dspace/config/spring/api/bitstore.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subfolder" value="${assetstore.s3.subfolder}"/>

<!-- Endpoint for other S3 providers than amazon AWS-->
<property name="endpoint" value="${assetstore.s3.endpoint}"/>
<property name="pathStyleAccessEnabled" value="${assetstore.s3.pathStyleAccessEnabled}"/>
</bean>

<!-- <bean name="localStore2 ... -->
Expand Down