Add third party integration tests for snapshot based recoveries

This commit adds third party integration tests for snapshot based
recoveries in S3, Azure and GCS.

Relates elastic#73496

qa/snapshot-based-recoveries/azure/build.gradle

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.internal.info.BuildParams
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+final Project fixture = project(':test:fixtures:azure-fixture')
+final Project repositoryPlugin = project(':plugins:repository-azure')
+
+dependencies {
+  testImplementation testArtifact(project(':qa:snapshot-based-recoveries'))
+  testImplementation repositoryPlugin
+}
+
+restResources {
+  restApi {
+    include 'indices', 'search', 'bulk', 'snapshot'
+  }
+}
+
+boolean useFixture = false
+String azureAccount = System.getenv("azure_storage_account")
+String azureKey = System.getenv("azure_storage_key")
+String azureContainer = System.getenv("azure_storage_container")
+String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
+
+if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
+  azureAccount = 'azure_integration_test_account'
+  azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
+  azureContainer = 'container'
+  azureBasePath = ''
+  azureSasToken = ''
+  useFixture = true
+
+}
+
+if (useFixture) {
+  apply plugin: 'elasticsearch.test.fixtures'
+  testFixtures.useFixture(fixture.path, 'azure-fixture-snapshot-based-recoveries')
+}
+
+tasks.named("integTest").configure {
+  systemProperty 'test.azure.container', azureContainer
+  nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_snapshot_based_recoveries_tests_" + BuildParams.testSeed
+}
+
+testClusters.matching { it.name == "integTest" }.configureEach {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 3
+  plugin repositoryPlugin.path
+
+  keystore 'azure.client.snapshot_based_recoveries.account', azureAccount
+  if (azureKey != null && azureKey.isEmpty() == false) {
+    keystore 'azure.client.snapshot_based_recoveries.key', azureKey
+  }
+  if (azureSasToken != null && azureSasToken.isEmpty() == false) {
+    keystore 'azure.client.snapshot_based_recoveries.sas_token', azureSasToken
+  }
+
+  setting 'xpack.security.enabled', 'false'
+
+  if (useFixture) {
+    def fixtureAddress = { fixtureName ->
+      assert useFixture: 'closure should not be used without a fixture'
+      int ephemeralPort =  fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.8091"
+      assert ephemeralPort > 0
+      '127.0.0.1:' + ephemeralPort
+    }
+    setting 'azure.client.snapshot_based_recoveries.endpoint_suffix',
+      { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-snapshot-based-recoveries')}/azure_integration_test_account" }, IGNORE_VALUE
+
+  } else {
+    println "Using an external service to test " + project.name
+  }
+}
+
+tasks.register("azureThirdPartyTest") {
+  dependsOn "integTest"
+}
+

qa/snapshot-based-recoveries/azure/src/test/java/org/elasticsearch/recovery/AzureSnapshotBasedRecoveryIT.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.recovery;
+
+import org.elasticsearch.common.settings.Settings;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return "azure";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String container = System.getProperty("test.azure.container");
+        assertThat(container, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.azure.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder()
+            .put("client", "snapshot_based_recoveries")
+            .put("container", container).put("base_path", basePath)
+            .build();
+    }
+}

qa/snapshot-based-recoveries/fs/build.gradle

@@ -1,3 +1,5 @@
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -6,11 +8,12 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.java-rest-test'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
-  javaRestTestImplementation(testArtifact(project(':qa:snapshot-based-recoveries')))
+  testImplementation testArtifact(project(':qa:snapshot-based-recoveries'))
 }
 
 final File repoDir = file("$buildDir/testclusters/snapshot-recoveries-repo")
@@ -28,7 +31,7 @@ tasks.withType(Test).configureEach {
   systemProperty 'tests.path.repo', repoDir
 }
 
-testClusters.all {
+testClusters.matching { it.name == "integTest" }.configureEach {
   testDistribution = 'DEFAULT'
   numberOfNodes = 3
   setting 'path.repo', repoDir.absolutePath

qa/snapshot-based-recoveries/gcs/build.gradle

@@ -0,0 +1,116 @@
+import org.apache.tools.ant.filters.ReplaceTokens
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+import java.security.KeyPair
+import java.security.KeyPairGenerator
+
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+final Project fixture = project(':test:fixtures:gcs-fixture')
+final Project repositoryPlugin = project(':plugins:repository-gcs')
+
+dependencies {
+  testImplementation testArtifact(project(':qa:snapshot-based-recoveries'))
+  testImplementation repositoryPlugin
+}
+
+restResources {
+  restApi {
+    include 'indices', 'search', 'bulk', 'snapshot'
+  }
+}
+
+boolean useFixture = false
+
+String gcsServiceAccount = System.getenv("google_storage_service_account")
+String gcsBucket = System.getenv("google_storage_bucket")
+String gcsBasePath = System.getenv("google_storage_base_path")
+
+File serviceAccountFile = null
+if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
+  serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
+  gcsBucket = 'bucket'
+  gcsBasePath = 'integration_test'
+  useFixture = true
+} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
+  throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
+} else {
+  serviceAccountFile = new File(gcsServiceAccount)
+}
+
+/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
+tasks.register("createServiceAccountFile") {
+  doLast {
+    KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
+    keyPairGenerator.initialize(2048)
+    KeyPair keyPair = keyPairGenerator.generateKeyPair()
+    String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
+
+    serviceAccountFile.parentFile.mkdirs()
+    serviceAccountFile.setText("{\n" +
+      '  "type": "service_account",\n' +
+      '  "project_id": "integration_test",\n' +
+      '  "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
+      '  "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
+      '  "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
+      '  "client_id": "123456789101112130594"\n' +
+      '}', 'UTF-8')
+  }
+}
+
+def fixtureAddress = { f ->
+  assert useFixture: 'closure should not be used without a fixture'
+  int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${f}.tcp.80"
+  assert ephemeralPort > 0
+  'http://127.0.0.1:' + ephemeralPort
+}
+
+Map<String, Object> expansions = [
+  'bucket'   : gcsBucket,
+  'base_path': gcsBasePath + "_integration_tests"
+]
+
+tasks.named("processTestResources").configure {
+  inputs.properties(expansions)
+  filter("tokens" : expansions, ReplaceTokens.class)
+}
+
+if (useFixture) {
+  apply plugin: 'elasticsearch.test.fixtures'
+  testFixtures.useFixture(fixture.path, 'gcs-fixture-snapshots-based-recoveries')
+}
+
+tasks.named("integTest").configure {
+  systemProperty 'test.gcs.bucket', gcsBucket
+  nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed
+
+  if (useFixture) {
+    dependsOn "createServiceAccountFile"
+  }
+}
+
+testClusters.matching { it.name == "integTest" }.configureEach {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 3
+  plugin repositoryPlugin.path
+
+  keystore 'gcs.client.snapshot_based_recoveries.credentials_file', serviceAccountFile, IGNORE_VALUE
+  if (useFixture) {
+    /* Use a closure on the string to delay evaluation until tests are executed */
+    setting 'gcs.client.snapshot_based_recoveries.endpoint', { "${-> fixtureAddress('gcs-fixture-snapshots-based-recoveries')}" }, IGNORE_VALUE
+    setting 'gcs.client.snapshot_based_recoveries.token_uri', { "${-> fixtureAddress('gcs-fixture-snapshots-based-recoveries')}/o/oauth2/token" },
+      IGNORE_VALUE
+  } else {
+    println "Using an external service to test " + project.name
+  }
+
+  setting 'xpack.security.enabled', 'false'
+}
+
+tasks.register("gcsThirdPartyTest")  {
+  dependsOn "integTest"
+}

qa/snapshot-based-recoveries/gcs/src/test/java/org/elasticsearch/recovery/GCSSnapshotBasedRecoveryIT.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.recovery;
+
+import org.elasticsearch.common.settings.Settings;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public class GCSSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return "gcs";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String bucket = System.getProperty("test.gcs.bucket");
+        assertThat(bucket, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.gcs.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder()
+            .put("client", "snapshot_based_recoveries")
+            .put("bucket", bucket).put("base_path", basePath)
+            .build();
+    }
+}

qa/snapshot-based-recoveries/s3/build.gradle

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+final Project fixture = project(':test:fixtures:s3-fixture')
+final Project repositoryPlugin = project(':plugins:repository-s3')
+
+dependencies {
+  testImplementation testArtifact(project(':qa:snapshot-based-recoveries'))
+  testImplementation repositoryPlugin
+}
+
+restResources {
+  restApi {
+    include 'indices', 'search', 'bulk', 'snapshot'
+  }
+}
+
+boolean useFixture = false
+String s3AccessKey = System.getenv("amazon_s3_access_key")
+String s3SecretKey = System.getenv("amazon_s3_secret_key")
+String s3Bucket = System.getenv("amazon_s3_bucket")
+String s3BasePath = System.getenv("amazon_s3_base_path")
+
+if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
+  s3AccessKey = 's3_test_access_key'
+  s3SecretKey = 's3_test_secret_key'
+  s3Bucket = 'bucket'
+  s3BasePath = null
+  useFixture = true
+
+} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
+  throw new IllegalArgumentException("not all options specified to run against external S3 service are present")
+}
+
+if (useFixture) {
+  apply plugin: 'elasticsearch.test.fixtures'
+  testFixtures.useFixture(fixture.path, 's3-snapshot-based-recoveries')
+}
+
+tasks.withType(Test).configureEach {
+  systemProperty 'test.s3.bucket', s3Bucket
+  nonInputProperties.systemProperty 'test.s3.base_path',
+    s3BasePath ? s3BasePath + "_snapshot_based_recoveries_tests" + BuildParams.testSeed : 'base_path'
+}
+
+testClusters.matching { it.name == "integTest" }.configureEach {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 3
+  plugin repositoryPlugin.path
+
+  keystore 's3.client.snapshot_based_recoveries.access_key', s3AccessKey
+  keystore 's3.client.snapshot_based_recoveries.secret_key', s3SecretKey
+
+  if (useFixture) {
+    def fixtureAddress = { fixtureName ->
+      assert useFixture: 'closure should not be used without a fixture'
+      int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
+      assert ephemeralPort > 0
+      '127.0.0.1:' + ephemeralPort
+    }
+    setting 's3.client.snapshot_based_recoveries.protocol', 'http'
+    setting 's3.client.snapshot_based_recoveries.endpoint', { "${-> fixtureAddress('s3-snapshot-based-recoveries')}" }, IGNORE_VALUE
+
+  } else {
+    println "Using an external service to test " + project.name
+  }
+
+  setting 'xpack.security.enabled', 'false'
+}
+
+tasks.register("s3ThirdPartyTest") {
+  dependsOn "integTest"
+}