From 4e010a11451dd9bb59158f9b46a71a5ab65f6314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Fern=C3=A1ndez=20Casta=C3=B1o?= Date: Thu, 12 Aug 2021 13:49:05 +0200 Subject: [PATCH] Add third party integration tests for snapshot based recoveries This commit adds third party integration tests for snapshot based recoveries in S3, Azure and GCS. Relates #73496 --- .../azure/build.gradle | 90 ++++++++++++++ .../AzureSnapshotBasedRecoveryIT.java | 36 ++++++ qa/snapshot-based-recoveries/fs/build.gradle | 9 +- .../recovery/FsSnapshotBasedRecoveryIT.java | 0 qa/snapshot-based-recoveries/gcs/build.gradle | 116 ++++++++++++++++++ .../recovery/GCSSnapshotBasedRecoveryIT.java | 36 ++++++ qa/snapshot-based-recoveries/s3/build.gradle | 85 +++++++++++++ .../recovery/S3SnapshotBasedRecoveryIT.java | 37 ++++++ .../fixtures/azure-fixture/docker-compose.yml | 9 ++ test/fixtures/gcs-fixture/docker-compose.yml | 12 ++ test/fixtures/s3-fixture/docker-compose.yml | 15 +++ 11 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 qa/snapshot-based-recoveries/azure/build.gradle create mode 100644 qa/snapshot-based-recoveries/azure/src/test/java/org/elasticsearch/recovery/AzureSnapshotBasedRecoveryIT.java rename qa/snapshot-based-recoveries/fs/src/{javaRestTest => test}/java/org/elasticsearch/recovery/FsSnapshotBasedRecoveryIT.java (100%) create mode 100644 qa/snapshot-based-recoveries/gcs/build.gradle create mode 100644 qa/snapshot-based-recoveries/gcs/src/test/java/org/elasticsearch/recovery/GCSSnapshotBasedRecoveryIT.java create mode 100644 qa/snapshot-based-recoveries/s3/build.gradle create mode 100644 qa/snapshot-based-recoveries/s3/src/test/java/org/elasticsearch/recovery/S3SnapshotBasedRecoveryIT.java diff --git a/qa/snapshot-based-recoveries/azure/build.gradle b/qa/snapshot-based-recoveries/azure/build.gradle new file mode 100644 index 0000000000000..2b1ea99d45bc1 --- /dev/null +++ b/qa/snapshot-based-recoveries/azure/build.gradle @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.internal.info.BuildParams +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:azure-fixture') +final Project repositoryPlugin = project(':plugins:repository-azure') + +dependencies { + testImplementation testArtifact(project(':qa:snapshot-based-recoveries')) + testImplementation repositoryPlugin +} + +restResources { + restApi { + include 'indices', 'search', 'bulk', 'snapshot' + } +} + +boolean useFixture = false +String azureAccount = System.getenv("azure_storage_account") +String azureKey = System.getenv("azure_storage_key") +String azureContainer = System.getenv("azure_storage_container") +String azureBasePath = System.getenv("azure_storage_base_path") +String azureSasToken = System.getenv("azure_storage_sas_token") + +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { + azureAccount = 'azure_integration_test_account' + azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 + azureContainer = 'container' + azureBasePath = '' + azureSasToken = '' + useFixture = true + +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 'azure-fixture-snapshot-based-recoveries') +} + +tasks.named("integTest").configure { + systemProperty 'test.azure.container', azureContainer + nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_snapshot_based_recoveries_tests_" + BuildParams.testSeed +} + +testClusters.matching { it.name == "integTest" }.configureEach { + testDistribution = 'DEFAULT' + numberOfNodes = 3 + plugin repositoryPlugin.path + + keystore 'azure.client.snapshot_based_recoveries.account', azureAccount + if (azureKey != null && azureKey.isEmpty() == false) { + keystore 'azure.client.snapshot_based_recoveries.key', azureKey + } + if (azureSasToken != null && azureSasToken.isEmpty() == false) { + keystore 'azure.client.snapshot_based_recoveries.sas_token', azureSasToken + } + + setting 'xpack.security.enabled', 'false' + + if (useFixture) { + def fixtureAddress = { fixtureName -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.8091" + assert ephemeralPort > 0 + '127.0.0.1:' + ephemeralPort + } + setting 'azure.client.snapshot_based_recoveries.endpoint_suffix', + { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-snapshot-based-recoveries')}/azure_integration_test_account" }, IGNORE_VALUE + + } else { + println "Using an external service to test " + project.name + } +} + +tasks.register("azureThirdPartyTest") { + dependsOn "integTest" +} + diff --git a/qa/snapshot-based-recoveries/azure/src/test/java/org/elasticsearch/recovery/AzureSnapshotBasedRecoveryIT.java b/qa/snapshot-based-recoveries/azure/src/test/java/org/elasticsearch/recovery/AzureSnapshotBasedRecoveryIT.java new file mode 100644 index 0000000000000..84f15c065e610 --- /dev/null +++ b/qa/snapshot-based-recoveries/azure/src/test/java/org/elasticsearch/recovery/AzureSnapshotBasedRecoveryIT.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.recovery; + +import org.elasticsearch.common.settings.Settings; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; + +public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase { + + @Override + protected String repositoryType() { + return "azure"; + } + + @Override + protected Settings repositorySettings() { + final String container = System.getProperty("test.azure.container"); + assertThat(container, not(blankOrNullString())); + + final String basePath = System.getProperty("test.azure.base_path"); + assertThat(basePath, not(blankOrNullString())); + + return Settings.builder() + .put("client", "snapshot_based_recoveries") + .put("container", container).put("base_path", basePath) + .build(); + } +} diff --git a/qa/snapshot-based-recoveries/fs/build.gradle b/qa/snapshot-based-recoveries/fs/build.gradle index b5cdb145379d7..224f43e18f44b 100644 --- a/qa/snapshot-based-recoveries/fs/build.gradle +++ b/qa/snapshot-based-recoveries/fs/build.gradle @@ -1,3 +1,5 @@ +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -6,11 +8,12 @@ * Side Public License, v 1. */ -apply plugin: 'elasticsearch.java-rest-test' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.rest-resources' dependencies { - javaRestTestImplementation(testArtifact(project(':qa:snapshot-based-recoveries'))) + testImplementation testArtifact(project(':qa:snapshot-based-recoveries')) } final File repoDir = file("$buildDir/testclusters/snapshot-recoveries-repo") @@ -28,7 +31,7 @@ tasks.withType(Test).configureEach { systemProperty 'tests.path.repo', repoDir } -testClusters.all { +testClusters.matching { it.name == "integTest" }.configureEach { testDistribution = 'DEFAULT' numberOfNodes = 3 setting 'path.repo', repoDir.absolutePath diff --git a/qa/snapshot-based-recoveries/fs/src/javaRestTest/java/org/elasticsearch/recovery/FsSnapshotBasedRecoveryIT.java b/qa/snapshot-based-recoveries/fs/src/test/java/org/elasticsearch/recovery/FsSnapshotBasedRecoveryIT.java similarity index 100% rename from qa/snapshot-based-recoveries/fs/src/javaRestTest/java/org/elasticsearch/recovery/FsSnapshotBasedRecoveryIT.java rename to qa/snapshot-based-recoveries/fs/src/test/java/org/elasticsearch/recovery/FsSnapshotBasedRecoveryIT.java diff --git a/qa/snapshot-based-recoveries/gcs/build.gradle b/qa/snapshot-based-recoveries/gcs/build.gradle new file mode 100644 index 0000000000000..a0dda78f6cf1b --- /dev/null +++ b/qa/snapshot-based-recoveries/gcs/build.gradle @@ -0,0 +1,116 @@ +import org.apache.tools.ant.filters.ReplaceTokens +import org.elasticsearch.gradle.internal.info.BuildParams + +import java.security.KeyPair +import java.security.KeyPairGenerator + +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:gcs-fixture') +final Project repositoryPlugin = project(':plugins:repository-gcs') + +dependencies { + testImplementation testArtifact(project(':qa:snapshot-based-recoveries')) + testImplementation repositoryPlugin +} + +restResources { + restApi { + include 'indices', 'search', 'bulk', 'snapshot' + } +} + +boolean useFixture = false + +String gcsServiceAccount = System.getenv("google_storage_service_account") +String gcsBucket = System.getenv("google_storage_bucket") +String gcsBasePath = System.getenv("google_storage_base_path") + +File serviceAccountFile = null +if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { + serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json') + gcsBucket = 'bucket' + gcsBasePath = 'integration_test' + useFixture = true +} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) { + throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present") +} else { + serviceAccountFile = new File(gcsServiceAccount) +} + +/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/ +tasks.register("createServiceAccountFile") { + doLast { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA") + keyPairGenerator.initialize(2048) + KeyPair keyPair = keyPairGenerator.generateKeyPair() + String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded()) + + serviceAccountFile.parentFile.mkdirs() + serviceAccountFile.setText("{\n" + + ' "type": "service_account",\n' + + ' "project_id": "integration_test",\n' + + ' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' + + ' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' + + ' "client_email": "integration_test@appspot.gserviceaccount.com",\n' + + ' "client_id": "123456789101112130594"\n' + + '}', 'UTF-8') + } +} + +def fixtureAddress = { f -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${f}.tcp.80" + assert ephemeralPort > 0 + 'http://127.0.0.1:' + ephemeralPort +} + +Map expansions = [ + 'bucket' : gcsBucket, + 'base_path': gcsBasePath + "_integration_tests" +] + +tasks.named("processTestResources").configure { + inputs.properties(expansions) + filter("tokens" : expansions, ReplaceTokens.class) +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 'gcs-fixture-snapshots-based-recoveries') +} + +tasks.named("integTest").configure { + systemProperty 'test.gcs.bucket', gcsBucket + nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed + + if (useFixture) { + dependsOn "createServiceAccountFile" + } +} + +testClusters.matching { it.name == "integTest" }.configureEach { + testDistribution = 'DEFAULT' + numberOfNodes = 3 + plugin repositoryPlugin.path + + keystore 'gcs.client.snapshot_based_recoveries.credentials_file', serviceAccountFile, IGNORE_VALUE + if (useFixture) { + /* Use a closure on the string to delay evaluation until tests are executed */ + setting 'gcs.client.snapshot_based_recoveries.endpoint', { "${-> fixtureAddress('gcs-fixture-snapshots-based-recoveries')}" }, IGNORE_VALUE + setting 'gcs.client.snapshot_based_recoveries.token_uri', { "${-> fixtureAddress('gcs-fixture-snapshots-based-recoveries')}/o/oauth2/token" }, + IGNORE_VALUE + } else { + println "Using an external service to test " + project.name + } + + setting 'xpack.security.enabled', 'false' +} + +tasks.register("gcsThirdPartyTest") { + dependsOn "integTest" +} diff --git a/qa/snapshot-based-recoveries/gcs/src/test/java/org/elasticsearch/recovery/GCSSnapshotBasedRecoveryIT.java b/qa/snapshot-based-recoveries/gcs/src/test/java/org/elasticsearch/recovery/GCSSnapshotBasedRecoveryIT.java new file mode 100644 index 0000000000000..ff623f31e5113 --- /dev/null +++ b/qa/snapshot-based-recoveries/gcs/src/test/java/org/elasticsearch/recovery/GCSSnapshotBasedRecoveryIT.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.recovery; + +import org.elasticsearch.common.settings.Settings; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; + +public class GCSSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase { + + @Override + protected String repositoryType() { + return "gcs"; + } + + @Override + protected Settings repositorySettings() { + final String bucket = System.getProperty("test.gcs.bucket"); + assertThat(bucket, not(blankOrNullString())); + + final String basePath = System.getProperty("test.gcs.base_path"); + assertThat(basePath, not(blankOrNullString())); + + return Settings.builder() + .put("client", "snapshot_based_recoveries") + .put("bucket", bucket).put("base_path", basePath) + .build(); + } +} diff --git a/qa/snapshot-based-recoveries/s3/build.gradle b/qa/snapshot-based-recoveries/s3/build.gradle new file mode 100644 index 0000000000000..642eaa3d1cca1 --- /dev/null +++ b/qa/snapshot-based-recoveries/s3/build.gradle @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE +import org.elasticsearch.gradle.internal.info.BuildParams + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project fixture = project(':test:fixtures:s3-fixture') +final Project repositoryPlugin = project(':plugins:repository-s3') + +dependencies { + testImplementation testArtifact(project(':qa:snapshot-based-recoveries')) + testImplementation repositoryPlugin +} + +restResources { + restApi { + include 'indices', 'search', 'bulk', 'snapshot' + } +} + +boolean useFixture = false +String s3AccessKey = System.getenv("amazon_s3_access_key") +String s3SecretKey = System.getenv("amazon_s3_secret_key") +String s3Bucket = System.getenv("amazon_s3_bucket") +String s3BasePath = System.getenv("amazon_s3_base_path") + +if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { + s3AccessKey = 's3_test_access_key' + s3SecretKey = 's3_test_secret_key' + s3Bucket = 'bucket' + s3BasePath = null + useFixture = true + +} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) { + throw new IllegalArgumentException("not all options specified to run against external S3 service are present") +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(fixture.path, 's3-snapshot-based-recoveries') +} + +tasks.withType(Test).configureEach { + systemProperty 'test.s3.bucket', s3Bucket + nonInputProperties.systemProperty 'test.s3.base_path', + s3BasePath ? s3BasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed : 'base_path' +} + +testClusters.matching { it.name == "integTest" }.configureEach { + testDistribution = 'DEFAULT' + numberOfNodes = 3 + plugin repositoryPlugin.path + + keystore 's3.client.snapshot_based_recoveries.access_key', s3AccessKey + keystore 's3.client.snapshot_based_recoveries.secret_key', s3SecretKey + + if (useFixture) { + def fixtureAddress = { fixtureName -> + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80" + assert ephemeralPort > 0 + '127.0.0.1:' + ephemeralPort + } + setting 's3.client.snapshot_based_recoveries.protocol', 'http' + setting 's3.client.snapshot_based_recoveries.endpoint', { "${-> fixtureAddress('s3-snapshot-based-recoveries')}" }, IGNORE_VALUE + + } else { + println "Using an external service to test " + project.name + } + + setting 'xpack.security.enabled', 'false' +} + +tasks.register("s3ThirdPartyTest") { + dependsOn "integTest" +} diff --git a/qa/snapshot-based-recoveries/s3/src/test/java/org/elasticsearch/recovery/S3SnapshotBasedRecoveryIT.java b/qa/snapshot-based-recoveries/s3/src/test/java/org/elasticsearch/recovery/S3SnapshotBasedRecoveryIT.java new file mode 100644 index 0000000000000..d5d72cd8489b0 --- /dev/null +++ b/qa/snapshot-based-recoveries/s3/src/test/java/org/elasticsearch/recovery/S3SnapshotBasedRecoveryIT.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.recovery; + +import org.elasticsearch.common.settings.Settings; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; + +public class S3SnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase { + + @Override + protected String repositoryType() { + return "s3"; + } + + @Override + protected Settings repositorySettings() { + final String bucket = System.getProperty("test.s3.bucket"); + assertThat(bucket, not(blankOrNullString())); + + final String basePath = System.getProperty("test.s3.base_path"); + assertThat(basePath, not(blankOrNullString())); + + return Settings.builder() + .put("client", "snapshot_based_recoveries") + .put("bucket", bucket) + .put("base_path", basePath) + .build(); + } +} diff --git a/test/fixtures/azure-fixture/docker-compose.yml b/test/fixtures/azure-fixture/docker-compose.yml index 2642efc5e8869..1b6d95848a744 100644 --- a/test/fixtures/azure-fixture/docker-compose.yml +++ b/test/fixtures/azure-fixture/docker-compose.yml @@ -35,3 +35,12 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "8091" + + azure-fixture-snapshot-based-recoveries: + build: + context: . + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "8091" diff --git a/test/fixtures/gcs-fixture/docker-compose.yml b/test/fixtures/gcs-fixture/docker-compose.yml index afe417319a95f..9144a8a763eb5 100644 --- a/test/fixtures/gcs-fixture/docker-compose.yml +++ b/test/fixtures/gcs-fixture/docker-compose.yml @@ -60,6 +60,18 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "80" + gcs-fixture-snapshots-based-recoveries: + build: + context: . + args: + port: 80 + bucket: "bucket" + token: "o/oauth2/token" + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" gcs-fixture-with-application-default-credentials: build: context: . diff --git a/test/fixtures/s3-fixture/docker-compose.yml b/test/fixtures/s3-fixture/docker-compose.yml index 0bcac68d99258..512e80bb8e0b0 100644 --- a/test/fixtures/s3-fixture/docker-compose.yml +++ b/test/fixtures/s3-fixture/docker-compose.yml @@ -60,6 +60,21 @@ services: ports: - "80" + s3-snapshot-based-recoveries: + build: + context: . + args: + fixtureClass: fixture.s3.S3HttpFixture + port: 80 + bucket: "bucket" + basePath: "base_path" + accessKey: "s3_test_access_key" + dockerfile: Dockerfile + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" + s3-fixture-with-session-token: build: context: .