Skip to content

Commit

Permalink
QA: Test template creation during rolling restart (#30850)
Browse files Browse the repository at this point in the history
Adds a test that we create the appropriate x-pack templates during the
rolling restart from the pre-6.2 OSS-zip distribution to the new zip
distribution that contains xpack. This is one way to answer the question
"does xpack acting sanely during the rolling upgrade and after it?" It
isn't as good as full exercising xpack but it is fairly simple and would
have caught #30832.

Relates to #30731
  • Loading branch information
nik9000 authored May 25, 2018
1 parent b4195d3 commit f561bad
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 18 deletions.
4 changes: 4 additions & 0 deletions qa/rolling-upgrade/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ for (Version version : bwcVersions.wireCompatible) {
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
oldClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'old_cluster'
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
}

Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
Expand All @@ -96,6 +97,7 @@ for (Version version : bwcVersions.wireCompatible) {
oneThirdUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'true'
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
}

Expand All @@ -108,6 +110,7 @@ for (Version version : bwcVersions.wireCompatible) {
twoThirdsUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'false'
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
}

Expand All @@ -119,6 +122,7 @@ for (Version version : bwcVersions.wireCompatible) {
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
/*
* Force stopping all the upgraded nodes after the test runner
* so they are alive during the test.
Expand Down
133 changes: 115 additions & 18 deletions qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,22 @@
package org.elasticsearch.upgrades;

import org.apache.http.util.EntityUtils;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.Before;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assume.assumeThat;

/**
Expand All @@ -36,11 +43,9 @@
*/
public class XPackIT extends AbstractRollingTestCase {
@Before
public void skipIfNotXPack() {
public void skipIfNotZip() {
assumeThat("test is only supported if the distribution contains xpack",
System.getProperty("tests.distribution"), equalTo("zip"));
assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
/*
* *Mostly* we want this for when we're upgrading from pre-6.3's
* zip distribution which doesn't contain xpack to post 6.3's zip
Expand All @@ -50,11 +55,81 @@ public void skipIfNotXPack() {
}

/**
* Test a basic feature (SQL) which doesn't require any trial license.
* Note that the test methods on this class can run in any order so we
* <strong>might</strong> have already installed a trial license.
* Tests that xpack is able to work itself into a sane state during the
* upgrade by testing that it is able to create all of the templates that
* it needs. This isn't a very strong assertion of sanity, but it is better
* than nothing and should catch a few sad cases.
* <p>
* The trouble is that when xpack isn't able to create the templates that
* it needs it retries over and over and over again. This can
* <strong>really</strong> slow things down. This test asserts that xpack
* was able to create the templates so it <strong>shouldn't</strong> be
* spinning trying to create things and slowing down the rest of the
* system.
*/
public void testIndexTemplatesCreated() throws Exception {
Version upgradeFromVersion =
Version.fromString(System.getProperty("tests.upgrade_from_version"));
boolean upgradeFromVersionHasXPack = upgradeFromVersion.onOrAfter(Version.V_6_3_0);
assumeFalse("this test doesn't really prove anything if the starting version has xpack and it is *much* more complex to maintain",
upgradeFromVersionHasXPack);
assumeFalse("since we're upgrading from a version without x-pack it won't have any templates",
CLUSTER_TYPE == ClusterType.OLD);

List<String> expectedTemplates = new ArrayList<>();
// Watcher creates its templates as soon as the first watcher node connects
expectedTemplates.add(".triggered_watches");
expectedTemplates.add(".watch-history-8");
expectedTemplates.add(".watches");
if (masterIsNewVersion()) {
// Everything else waits until the master is upgraded to create its templates
expectedTemplates.add(".ml-anomalies-");
expectedTemplates.add(".ml-meta");
expectedTemplates.add(".ml-notifications");
expectedTemplates.add(".ml-state");
expectedTemplates.add("logstash-index-template");
expectedTemplates.add("security-index-template");
expectedTemplates.add("security_audit_log");
}
Collections.sort(expectedTemplates);

/*
* The index templates are created asynchronously after startup and
* while this is usually fast we use assertBusy here just in case
* they aren't created by the time this test is run.
*/
assertBusy(() -> {
List<String> actualTemplates;
try (XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
client().performRequest(new Request("GET", "/_template")).getEntity().getContent())) {
actualTemplates = new ArrayList<>(parser.map().keySet());
}
Collections.sort(actualTemplates);
/*
* This test asserts that the templates match *exactly* to force
* us to keep the list of templates up to date. Most templates
* aren't likely to cause a problem on upgrade but it is better
* to be safe and make sure they are all created than to be sorry
* and miss a bug that causes one to be missed on upgrade.
*
* We sort the templates so the error message is easy to read.
*/
assertEquals(expectedTemplates, actualTemplates);
});
}

/**
* Test a basic feature (SQL) after the upgrade which only requires the
* "default" basic license. Note that the test methods on this class can
* run in any order so we <strong>might</strong> have already installed a
* trial license.
*/
public void testBasicFeature() throws IOException {
public void testBasicFeatureAfterUpgrade() throws IOException {
assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));

Request bulk = new Request("POST", "/sql_test/doc/_bulk");
bulk.setJsonEntity(
"{\"index\":{}}\n"
Expand All @@ -71,16 +146,20 @@ public void testBasicFeature() throws IOException {
}

/**
* Test creating a trial license and using it. This is interesting because
* our other tests test cover starting a new cluster with the default
* distribution and enabling the trial license but this test is the only
* one that can upgrade from the oss distribution to the default
* distribution with xpack and the create a trial license. We don't
* <strong>do</strong> a lot with the trial license because for the most
* part those things are tested elsewhere, off in xpack. But we do use the
* trial license a little bit to make sure that it works.
* Test creating a trial license after the upgrade and a feature (ML) that
* requires the license. Our other tests test cover starting a new cluster
* with the default distribution and enabling the trial license but this
* test is the only one tests the rolling upgrade from the oss distribution
* to the default distribution with xpack and then creating of a trial
* license. We don't <strong>do</strong> a lot with the trial license
* because for the most part those things are tested elsewhere, off in
* xpack. But we do use the trial license a little bit to make sure that
* creating it worked properly.
*/
public void testTrialLicense() throws IOException {
assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));

Request startTrial = new Request("POST", "/_xpack/license/start_trial");
startTrial.addParameter("acknowledge", "true");
client().performRequest(startTrial);
Expand Down Expand Up @@ -108,4 +187,22 @@ public void testTrialLicense() throws IOException {
+ "}\n");
client().performRequest(createJob);
}

/**
* Has the master been upgraded to the new version?
*/
private boolean masterIsNewVersion() throws IOException {
Map<?, ?> map;
try (XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
client().performRequest(new Request("GET", "/_nodes/_master")).getEntity().getContent())) {
map = parser.map();
}
map = (Map<?, ?>) map.get("nodes");
assertThat(map.values(), hasSize(1));
map = (Map<?, ?>) map.values().iterator().next();
Version masterVersion = Version.fromString(map.get("version").toString());
return Version.CURRENT.equals(masterVersion);
}
}

0 comments on commit f561bad

Please sign in to comment.