diff --git a/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle b/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle index 12350bb29567a..87d4be3c11d18 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle @@ -13,6 +13,7 @@ * build/distributions/local * */ import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.VersionProperties // gradle has an open issue of failing applying plugins in // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004) @@ -29,6 +30,6 @@ tasks.register('localDistro', Sync) { from(elasticsearch_distributions.local) into("build/distribution/local") doLast { - logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}.") + logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}/elasticsearch-${VersionProperties.elasticsearch}") } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index d9c688d0ba2ea..a567159958451 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -102,7 +102,7 @@ public void apply(Project project) { Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest"); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); - // Configuration examplePlugin = configureExamplePlugin(project); + Configuration examplePlugin = configureExamplePlugin(project); List> windowsTestTasks = new ArrayList<>(); Map>> linuxTestTasks = new HashMap<>(); @@ -114,11 +114,12 @@ public void apply(Project project) { TaskProvider depsTask = project.getTasks().register(taskname + "#deps"); // explicitly depend on the archive not on the implicit extracted distribution depsTask.configure(t -> t.dependsOn(distribution.getArchiveDependencies())); + depsTask.configure(t -> t.dependsOn(examplePlugin.getDependencies())); depsTasks.put(taskname, depsTask); TaskProvider destructiveTask = configureTestTask(project, taskname, distribution, t -> { t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable); addSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath); - // addSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); + addSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString()); t.exclude("**/PackageUpgradeTests.class"); }, depsTask); @@ -313,7 +314,7 @@ private static Configuration configureExamplePlugin(Project project) { Configuration examplePlugin = project.getConfigurations().create(EXAMPLE_PLUGIN_CONFIGURATION); examplePlugin.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); DependencyHandler deps = project.getDependencies(); - deps.add(EXAMPLE_PLUGIN_CONFIGURATION, deps.create("org.elasticsearch.examples:custom-settings:1.0.0-SNAPSHOT")); + deps.add(EXAMPLE_PLUGIN_CONFIGURATION, deps.project(Map.of("path", ":plugins:analysis-icu", "configuration", "zip"))); return examplePlugin; } diff --git a/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml b/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml index 6aeae3712aaf9..1c966943c8ba2 100644 --- a/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml +++ b/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml @@ -34,7 +34,7 @@ - + diff --git a/distribution/build.gradle b/distribution/build.gradle index d00bd0f4fbc00..dbd20947ffa47 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -296,7 +296,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { *****************************************************************************/ libFiles = { testDistro -> copySpec { - // delay by using closures, since they have not yet been configured, so no jar task exists yet + // Delay by using closures, since they have not yet been configured, so no jar task exists yet. from(configurations.libs) into('tools/geoip-cli') { from(configurations.libsGeoIpCli) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index ba111e7298e06..954f5c411c60c 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -268,12 +268,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) { } // For some reason, the artifact name can differ depending on what repository we used. rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" - - into('bin') { - from(project.projectDir.toPath().resolve('src/docker/cloud')) { - expand([ version: VersionProperties.elasticsearch ]) - } - } } onlyIf { Architecture.current() == architecture } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index d102f353156e9..882eca2fd28ef 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -110,12 +110,17 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas find config -type f -exec chmod 0664 {} + <% if (docker_base == "cloud") { %> -# Preinstall common plugins +# Preinstall common plugins. Note that these are installed as root, meaning the `elasticsearch` user cannot delete them. COPY repository-s3-${version}.zip repository-gcs-${version}.zip repository-azure-${version}.zip /tmp/ -RUN bin/elasticsearch-plugin install --batch \\ +RUN bin/elasticsearch-plugin install --batch --verbose \\ file:/tmp/repository-s3-${version}.zip \\ file:/tmp/repository-gcs-${version}.zip \\ file:/tmp/repository-azure-${version}.zip +# Generate a replacement example plugins config that reflects what is actually installed +RUN echo "plugins:" > config/elasticsearch-plugins.example.yml && \\ + echo " - id: repository-azure" >> config/elasticsearch-plugins.example.yml && \\ + echo " - id: repository-gcs" >> config/elasticsearch-plugins.example.yml && \\ + echo " - id: repository-s3" >> config/elasticsearch-plugins.example.yml <% /* I tried to use `ADD` here, but I couldn't force it to do what I wanted */ %> COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ @@ -125,8 +130,6 @@ RUN mkdir -p /opt/filebeat /opt/metricbeat && \\ # Add plugins infrastructure RUN mkdir -p /opt/plugins/archive -COPY bin/plugin-wrapper.sh /opt/plugins -# These are the correct permissions for both the directories and the script RUN chmod -R 0555 /opt/plugins <% } %> diff --git a/distribution/docker/src/docker/Dockerfile.cloud-ess b/distribution/docker/src/docker/Dockerfile.cloud-ess index 783dfc20d98fb..f82752d67a284 100644 --- a/distribution/docker/src/docker/Dockerfile.cloud-ess +++ b/distribution/docker/src/docker/Dockerfile.cloud-ess @@ -10,3 +10,4 @@ RUN chmod 0444 /opt/plugins/archive/* FROM ${base_image} COPY --from=builder /opt/plugins /opt/plugins +ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/docker/src/docker/cloud/plugin-wrapper.sh b/distribution/docker/src/docker/cloud/plugin-wrapper.sh deleted file mode 100755 index 248ffc7a91ade..0000000000000 --- a/distribution/docker/src/docker/cloud/plugin-wrapper.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -<% /* Populated by Gradle */ %> -VERSION="$version" - -plugin_name_is_next=0 - -declare -a args_array - -while test \$# -gt 0; do - opt="\$1" - shift - - if [[ \$plugin_name_is_next -eq 1 ]]; then - if [[ -f "/opt/plugins/archive/\$opt-\${VERSION}.zip" ]]; then - opt="file:/opt/plugins/archive/\$opt-\${VERSION}.zip" - fi - elif [[ "\$opt" == "install" ]]; then - plugin_name_is_next=1 - fi - - args_array+=("\$opt") -done - -set -- "\$@" "\${args_array[@]}" - -exec /usr/share/elasticsearch/bin/elasticsearch-plugin "\$@" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index e2739f07e2aeb..cfc987c79caa6 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -205,6 +205,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk, String archit // ========= config files ========= configurationFile '/etc/elasticsearch/elasticsearch.yml' + configurationFile '/etc/elasticsearch/elasticsearch-plugins.example.yml' configurationFile '/etc/elasticsearch/jvm.options' configurationFile '/etc/elasticsearch/log4j2.properties' if (oss == false) { diff --git a/distribution/src/config/elasticsearch-plugins.example.yml b/distribution/src/config/elasticsearch-plugins.example.yml new file mode 100644 index 0000000000000..b6874e915feec --- /dev/null +++ b/distribution/src/config/elasticsearch-plugins.example.yml @@ -0,0 +1,27 @@ +# Rename this file to `elasticsearch-plugins.yml` to use it. +# +# All plugins must be listed here. If you add a plugin to this list and run +# `elasticsearch-plugin sync`, that plugin will be installed. If you remove +# a plugin from this list, that plugin will be removed when Elasticsearch +# next starts. + +plugins: + # Each plugin must have an ID. Plugins with only an ID are official plugins and will be downloaded from Elastic. + - id: example-id + + # Plugins can be specified by URL (it doesn't have to be HTTP, you could use e.g. `file:`) + - id: example-with-url + location: https://some.domain/path/example4.zip + + # Or by maven coordinates: + - id: example-with-maven-url + location: org.elasticsearch.plugins:example-plugin:1.2.3 + + # A proxy can also be configured per-plugin, if necessary + - id: example-with-proxy + location: https://some.domain/path/example.zip + proxy: https://some.domain:1234 + +# Configures a proxy for all network access. Remove this if you don't need +# to use a proxy. +proxy: https://some.domain:1234 diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 2e545ed3401e8..d1a53a0e11bd8 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -13,10 +13,6 @@ archivesBaseName = 'elasticsearch-plugin-cli' dependencies { compileOnly project(":server") compileOnly project(":libs:elasticsearch-cli") - api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" api "org.bouncycastle:bcpg-fips:1.0.4" api "org.bouncycastle:bc-fips:1.0.2" testImplementation project(":test:framework") diff --git a/distribution/tools/plugin-cli/licenses/jackson-annotations-2.10.4.jar.sha1 b/distribution/tools/plugin-cli/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0e7711..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/jackson-annotations-LICENSE b/distribution/tools/plugin-cli/licenses/jackson-annotations-LICENSE deleted file mode 100644 index ff94ef8c456a6..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-annotations-LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor annotations is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/distribution/tools/plugin-cli/licenses/jackson-annotations-NOTICE.txt b/distribution/tools/plugin-cli/licenses/jackson-annotations-NOTICE.txt deleted file mode 100644 index 5ab1e5636037e..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-annotations-NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. diff --git a/distribution/tools/plugin-cli/licenses/jackson-databind-2.10.4.jar.sha1 b/distribution/tools/plugin-cli/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72cd27af..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/jackson-databind-LICENSE b/distribution/tools/plugin-cli/licenses/jackson-databind-LICENSE deleted file mode 100644 index 6acf75483f9b0..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-databind-LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor databind module is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/distribution/tools/plugin-cli/licenses/jackson-databind-NOTICE.txt b/distribution/tools/plugin-cli/licenses/jackson-databind-NOTICE.txt deleted file mode 100644 index 5ab1e5636037e..0000000000000 --- a/distribution/tools/plugin-cli/licenses/jackson-databind-NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index 94e647b275f38..1fcafa1f8ce1a 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; @@ -46,6 +47,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.net.HttpURLConnection; +import java.net.Proxy; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -113,7 +115,7 @@ * elasticsearch config directory, using the name of the plugin. If any files to be installed * already exist, they will be skipped. */ -class InstallPluginAction implements Closeable { +public class InstallPluginAction implements Closeable { private static final String PROPERTY_STAGING_ID = "es.plugins.staging"; @@ -150,7 +152,7 @@ class InstallPluginAction implements Closeable { } /** The official plugins that can be installed simply by name. */ - static final Set OFFICIAL_PLUGINS; + public static final Set OFFICIAL_PLUGINS; static { try ( InputStream stream = InstallPluginAction.class.getResourceAsStream("/plugins.txt"); @@ -198,15 +200,20 @@ class InstallPluginAction implements Closeable { private final Terminal terminal; private Environment env; private boolean batch; + private Proxy proxy = null; - InstallPluginAction(Terminal terminal, Environment env, boolean batch) { + public InstallPluginAction(Terminal terminal, Environment env, boolean batch) { this.terminal = terminal; this.env = env; this.batch = batch; } + public void setProxy(Proxy proxy) { + this.proxy = proxy; + } + // pkg private for testing - void execute(List plugins) throws Exception { + public void execute(List plugins) throws Exception { if (plugins.isEmpty()) { throw new UserException(ExitCodes.USAGE, "at least one plugin id is required"); } @@ -218,10 +225,12 @@ void execute(List plugins) throws Exception { } } + final String logPrefix = terminal.isHeadless() ? "" : "-> "; + final Map> deleteOnFailures = new LinkedHashMap<>(); for (final PluginDescriptor plugin : plugins) { final String pluginId = plugin.getId(); - terminal.println("-> Installing " + pluginId); + terminal.println(logPrefix + "Installing " + pluginId); try { if ("x-pack".equals(pluginId)) { handleInstallXPack(buildFlavor()); @@ -233,15 +242,15 @@ void execute(List plugins) throws Exception { final Path pluginZip = download(plugin, env.tmpFile()); final Path extractedZip = unzip(pluginZip, env.pluginsFile()); deleteOnFailure.add(extractedZip); - final PluginInfo pluginInfo = installPlugin(extractedZip, deleteOnFailure); - terminal.println("-> Installed " + pluginInfo.getName()); + final PluginInfo pluginInfo = installPlugin(plugin, extractedZip, deleteOnFailure); + terminal.println(logPrefix + "Installed " + pluginInfo.getName()); // swap the entry by plugin id for one with the installed plugin name, it gives a cleaner error message for URL installs deleteOnFailures.remove(pluginId); deleteOnFailures.put(pluginInfo.getName(), deleteOnFailure); } catch (final Exception installProblem) { - terminal.println("-> Failed installing " + pluginId); + terminal.println(logPrefix + "Failed installing " + pluginId); for (final Map.Entry> deleteOnFailureEntry : deleteOnFailures.entrySet()) { - terminal.println("-> Rolling back " + deleteOnFailureEntry.getKey()); + terminal.println(logPrefix + "Rolling back " + deleteOnFailureEntry.getKey()); boolean success = false; try { IOUtils.rm(deleteOnFailureEntry.getValue().toArray(new Path[0])); @@ -252,16 +261,18 @@ void execute(List plugins) throws Exception { exceptionWhileRemovingFiles ); installProblem.addSuppressed(exception); - terminal.println("-> Failed rolling back " + deleteOnFailureEntry.getKey()); + terminal.println(logPrefix + "Failed rolling back " + deleteOnFailureEntry.getKey()); } if (success) { - terminal.println("-> Rolled back " + deleteOnFailureEntry.getKey()); + terminal.println(logPrefix + "Rolled back " + deleteOnFailureEntry.getKey()); } } throw installProblem; } } - terminal.println("-> Please restart Elasticsearch to activate any plugins installed"); + if (terminal.isHeadless() == false) { + terminal.println("-> Please restart Elasticsearch to activate any plugins installed"); + } } Build.Flavor buildFlavor() { @@ -288,24 +299,37 @@ private static void handleInstallXPack(final Build.Flavor flavor) throws UserExc private Path download(PluginDescriptor plugin, Path tmpDir) throws Exception { final String pluginId = plugin.getId(); - if (OFFICIAL_PLUGINS.contains(pluginId)) { + final String logPrefix = terminal.isHeadless() ? "" : "-> "; + + // See `InstallPluginCommand` it has to use a string argument for both the ID and the location + if (OFFICIAL_PLUGINS.contains(pluginId) && (plugin.getLocation() == null || plugin.getLocation().equals(pluginId))) { + final String pluginArchiveDir = System.getenv("ES_PLUGIN_ARCHIVE_DIR"); + if (pluginArchiveDir != null && pluginArchiveDir.isEmpty() == false) { + final Path pluginPath = getPluginArchivePath(pluginId, pluginArchiveDir); + if (Files.exists(pluginPath)) { + terminal.println(logPrefix + "Downloading " + pluginId + " from local archive: " + pluginArchiveDir); + return downloadZip("file://" + pluginPath, tmpDir); + } + // else carry on to regular download + } + final String url = getElasticUrl(getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME); - terminal.println("-> Downloading " + pluginId + " from elastic"); + terminal.println(logPrefix + "Downloading " + pluginId + " from elastic"); return downloadAndValidate(url, tmpDir, true); } - final String pluginUrl = plugin.getUrl(); + final String pluginLocation = plugin.getLocation(); // now try as maven coordinates, a valid URL would only have a colon and slash - String[] coordinates = pluginUrl.split(":"); - if (coordinates.length == 3 && pluginUrl.contains("/") == false && pluginUrl.startsWith("file:") == false) { - String mavenUrl = getMavenUrl(coordinates, Platforms.PLATFORM_NAME); - terminal.println("-> Downloading " + pluginId + " from maven central"); + String[] coordinates = pluginLocation.split(":"); + if (coordinates.length == 3 && pluginLocation.contains("/") == false && pluginLocation.startsWith("file:") == false) { + String mavenUrl = getMavenUrl(coordinates); + terminal.println(logPrefix + "Downloading " + pluginId + " from maven central"); return downloadAndValidate(mavenUrl, tmpDir, false); } // fall back to plain old URL - if (pluginUrl.contains(":") == false) { + if (pluginLocation.contains(":") == false) { // definitely not a valid url, so assume it is a plugin name List pluginSuggestions = checkMisspelledPlugin(pluginId); String msg = "Unknown plugin " + pluginId; @@ -314,8 +338,20 @@ private Path download(PluginDescriptor plugin, Path tmpDir) throws Exception { } throw new UserException(ExitCodes.USAGE, msg); } - terminal.println("-> Downloading " + URLDecoder.decode(pluginUrl, StandardCharsets.UTF_8.name())); - return downloadZip(pluginUrl, tmpDir); + terminal.println(logPrefix + "Downloading " + URLDecoder.decode(pluginLocation, StandardCharsets.UTF_8.name())); + return downloadZip(pluginLocation, tmpDir); + } + + @SuppressForbidden(reason = "Need to use PathUtils#get") + private Path getPluginArchivePath(String pluginId, String pluginArchiveDir) throws UserException { + final Path path = PathUtils.get(pluginArchiveDir); + if (Files.exists(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Location in ES_PLUGIN_ARCHIVE_DIR does not exist"); + } + if (Files.isDirectory(path) == false) { + throw new UserException(ExitCodes.CONFIG, "Location in ES_PLUGIN_ARCHIVE_DIR is not a directory"); + } + return PathUtils.get(pluginArchiveDir, pluginId + "-" + Version.CURRENT + (isSnapshot() ? "-SNAPSHOT" : "") + ".zip"); } // pkg private so tests can override @@ -381,12 +417,12 @@ private String nonReleaseUrl(final String hostname, final Version version, final /** * Returns the url for an elasticsearch plugin in maven. */ - private String getMavenUrl(String[] coordinates, String platform) throws IOException { + private String getMavenUrl(String[] coordinates) throws IOException { final String groupId = coordinates[0].replace(".", "/"); final String artifactId = coordinates[1]; final String version = coordinates[2]; final String baseUrl = String.format(Locale.ROOT, "https://repo1.maven.org/maven2/%s/%s/%s", groupId, artifactId, version); - final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, artifactId, platform, version); + final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, artifactId, Platforms.PLATFORM_NAME, version); if (urlExists(platformUrl)) { return platformUrl; } @@ -424,7 +460,7 @@ private List checkMisspelledPlugin(String pluginId) { } } CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1())); - return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList()); + return scoredKeys.stream().map(Tuple::v2).collect(Collectors.toList()); } /** Downloads a zip from the url, into a temp file under the given temp dir. */ @@ -434,10 +470,10 @@ Path downloadZip(String urlString, Path tmpDir) throws IOException { terminal.println(VERBOSE, "Retrieving zip from " + urlString); URL url = new URL(urlString); Path zip = Files.createTempFile(tmpDir, null, ".zip"); - URLConnection urlConnection = url.openConnection(); + URLConnection urlConnection = this.proxy == null ? url.openConnection() : url.openConnection(this.proxy); urlConnection.addRequestProperty("User-Agent", "elasticsearch-plugin-installer"); try ( - InputStream in = batch + InputStream in = batch || terminal.isHeadless() ? urlConnection.getInputStream() : new TerminalProgressInputStream(urlConnection.getInputStream(), urlConnection.getContentLength(), terminal) ) { @@ -460,10 +496,10 @@ void setBatch(boolean batch) { /** * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0 */ - private class TerminalProgressInputStream extends ProgressInputStream { + private static class TerminalProgressInputStream extends ProgressInputStream { + private static final int WIDTH = 50; private final Terminal terminal; - private int width = 50; private final boolean enabled; TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) { @@ -475,13 +511,13 @@ private class TerminalProgressInputStream extends ProgressInputStream { @Override public void onProgress(int percent) { if (enabled) { - int currentPosition = percent * width / 100; + int currentPosition = percent * WIDTH / 100; StringBuilder sb = new StringBuilder("\r["); sb.append(String.join("=", Collections.nCopies(currentPosition, ""))); if (currentPosition > 0 && percent < 100) { sb.append(">"); } - sb.append(String.join(" ", Collections.nCopies(width - currentPosition, ""))); + sb.append(String.join(" ", Collections.nCopies(WIDTH - currentPosition, ""))); sb.append("] %s   "); if (percent == 100) { sb.append("\n"); @@ -493,7 +529,7 @@ public void onProgress(int percent) { @SuppressForbidden(reason = "URL#openStream") private InputStream urlOpenStream(final URL url) throws IOException { - return url.openStream(); + return this.proxy == null ? url.openStream() : url.openConnection(proxy).getInputStream(); } /** @@ -545,14 +581,10 @@ private Path downloadAndValidate(final String urlString, final Path tmpDir, fina * matches, and that the file contains a single line. For SHA-512, we verify that the hash and the filename match, and that the * file contains a single line. */ + final BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); if (digestAlgo.equals("SHA-1")) { - final BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); expectedChecksum = checksumReader.readLine(); - if (checksumReader.readLine() != null) { - throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); - } } else { - final BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); final String checksumLine = checksumReader.readLine(); final String[] fields = checksumLine.split(" {2}"); if (officialPlugin && fields.length != 2 || officialPlugin == false && fields.length > 2) { @@ -574,9 +606,9 @@ private Path downloadAndValidate(final String urlString, final Path tmpDir, fina throw new UserException(ExitCodes.IO_ERROR, message); } } - if (checksumReader.readLine() != null) { - throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); - } + } + if (checksumReader.readLine() != null) { + throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl); } } @@ -615,7 +647,7 @@ private Path downloadAndValidate(final String urlString, final Path tmpDir, fina * ".asc" to the URL. It is expected that the plugin is signed with the Elastic signing key with ID D27D666CD88E42B4. * * @param zip the path to the downloaded plugin ZIP - * @param urlString the URL source of the downloade plugin ZIP + * @param urlString the URL source of the downloaded plugin ZIP * @throws IOException if an I/O exception occurs reading from various input streams * @throws PGPException if the PGP implementation throws an internal exception during verification */ @@ -688,12 +720,14 @@ InputStream getPublicKey() { /** * Creates a URL and opens a connection. *

- * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned. + * If the URL returns a 404, {@code null} is returned, otherwise the open URL object is returned. */ // pkg private for tests URL openUrl(String urlString) throws IOException { URL checksumUrl = new URL(urlString); - HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection(); + HttpURLConnection connection = this.proxy == null + ? (HttpURLConnection) checksumUrl.openConnection() + : (HttpURLConnection) checksumUrl.openConnection(this.proxy); if (connection.getResponseCode() == 404) { return null; } @@ -849,7 +883,7 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidateDir, Path pluginsDir, * Installs the plugin from {@code tmpRoot} into the plugins dir. * If the plugin has a bin dir and/or a config dir, those are moved. */ - private PluginInfo installPlugin(Path tmpRoot, List deleteOnFailure) throws Exception { + private PluginInfo installPlugin(PluginDescriptor descriptor, Path tmpRoot, List deleteOnFailure) throws Exception { final PluginInfo info = loadPluginInfo(tmpRoot); checkCanInstallationProceed(terminal, Build.CURRENT.flavor(), info); PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpFile()); @@ -858,6 +892,16 @@ private PluginInfo installPlugin(Path tmpRoot, List deleteOnFailure) throw PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch); } + // Validate that the downloaded plugin's ID matches what we expect from the descriptor. The + // exception is if we install a plugin via `InstallPluginCommand` by specifying a URL or + // Maven coordinates, because then we can't know in advance what the plugin ID ought to be. + if (descriptor.getId().contains(":") == false && descriptor.getId().equals(info.getName()) == false) { + throw new UserException( + PLUGIN_MALFORMED, + "Expected downloaded plugin to have ID [" + descriptor.getId() + "] but found [" + info.getName() + "]" + ); + } + final Path destination = env.pluginsFile().resolve(info.getName()); deleteOnFailure.add(destination); @@ -893,7 +937,7 @@ private void installPluginSupportFiles(PluginInfo info, Path tmpRoot, Path destB /** * Moves the plugin directory into its final destination. - **/ + */ private void movePlugin(Path tmpRoot, Path destination) throws IOException { Files.move(tmpRoot, destination, StandardCopyOption.ATOMIC_MOVE); Files.walkFileTree(destination, new SimpleFileVisitor() { @@ -1008,10 +1052,10 @@ private static void setFileAttributes(final Path path, final Set plugins = arguments.values(options) .stream() - .map(id -> new PluginDescriptor(id, id)) + // We only have one piece of data, which could be an ID or could be a location, so we use it for both + .map(idOrLocation -> new PluginDescriptor(idOrLocation, idOrLocation)) .collect(Collectors.toList()); final boolean isBatch = options.has(batchOption); - InstallPluginAction action = new InstallPluginAction(terminal, env, isBatch); - action.execute(plugins); + try (InstallPluginAction action = new InstallPluginAction(terminal, env, isBatch)) { + action.execute(plugins); + } } } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java index 290771e7a4fc1..aebb33447c0f4 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java @@ -24,6 +24,8 @@ import java.util.Collections; import java.util.List; +import static org.elasticsearch.plugins.cli.SyncPluginsAction.ELASTICSEARCH_PLUGINS_YML_CACHE; + /** * A command for the plugin cli to list plugins installed in elasticsearch. */ @@ -42,8 +44,10 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile()); final List plugins = new ArrayList<>(); try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsFile())) { - for (Path plugin : paths) { - plugins.add(plugin); + for (Path path : paths) { + if (path.getFileName().toString().equals(ELASTICSEARCH_PLUGINS_YML_CACHE) == false) { + plugins.add(path); + } } } Collections.sort(plugins); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginDescriptor.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginDescriptor.java index fcee3709963fa..06b054716732e 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginDescriptor.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginDescriptor.java @@ -10,25 +10,29 @@ import java.util.Objects; +/** + * Models a single plugin that can be installed. + */ public class PluginDescriptor { private String id; - private String url; - private String proxy; + private String location; public PluginDescriptor() {} - public PluginDescriptor(String id, String url, String proxy) { - this.id = id; - this.url = url; - this.proxy = proxy; - } - - public PluginDescriptor(String id, String url) { - this(id, url, null); + /** + * Creates a new descriptor instance. + * + * @param id the name of the plugin. Cannot be null. + * @param location the location from which to fetch the plugin, e.g. a URL or Maven + * coordinates. Can be null for official plugins. + */ + public PluginDescriptor(String id, String location) { + this.id = Objects.requireNonNull(id, "id cannot be null"); + this.location = location; } public PluginDescriptor(String id) { - this(id, null, null); + this(id, null); } public String getId() { @@ -39,20 +43,12 @@ public void setId(String id) { this.id = id; } - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; + public String getLocation() { + return location; } - public String getProxy() { - return proxy; - } - - public void setProxy(String proxy) { - this.proxy = proxy; + public void setLocation(String location) { + this.location = location; } @Override @@ -60,11 +56,16 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PluginDescriptor that = (PluginDescriptor) o; - return id.equals(that.id) && Objects.equals(url, that.url) && Objects.equals(proxy, that.proxy); + return id.equals(that.id) && Objects.equals(location, that.location); } @Override public int hashCode() { - return Objects.hash(id, url, proxy); + return Objects.hash(id, location); + } + + @Override + public String toString() { + return "PluginDescriptor{id='" + id + "', location='" + location + "'}"; } } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java index 6efef209b73c1..79b45292c5476 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSecurity.java @@ -27,6 +27,10 @@ import java.util.Set; import java.util.stream.Collectors; +/** + * Contains methods for displaying extended plugin permissions to the user, and confirming that + * plugin installation can proceed. + */ public class PluginSecurity { /** @@ -37,30 +41,45 @@ static void confirmPolicyExceptions(Terminal terminal, Set permissions, if (requested.isEmpty()) { terminal.println(Verbosity.VERBOSE, "plugin has a policy file with no additional permissions"); } else { - // sort permissions in a reasonable order Collections.sort(requested); - terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); - terminal.errorPrintln(Verbosity.NORMAL, "@ WARNING: plugin requires additional permissions @"); - terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); - // print all permissions: - for (String permission : requested) { - terminal.errorPrintln(Verbosity.NORMAL, "* " + permission); + if (terminal.isHeadless()) { + terminal.errorPrintln( + "WARNING: plugin requires additional permissions: [" + + requested.stream().map(each -> '\'' + each + '\'').collect(Collectors.joining(", ")) + + "]" + ); + terminal.errorPrintln( + "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html" + + " for descriptions of what these permissions allow and the associated risks." + ); + } else { + terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); + terminal.errorPrintln(Verbosity.NORMAL, "@ WARNING: plugin requires additional permissions @"); + terminal.errorPrintln(Verbosity.NORMAL, "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"); + // print all permissions: + for (String permission : requested) { + terminal.errorPrintln(Verbosity.NORMAL, "* " + permission); + } + terminal.errorPrintln( + Verbosity.NORMAL, + "See https://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html" + ); + terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks."); + + if (batch == false) { + prompt(terminal); + } } - terminal.errorPrintln(Verbosity.NORMAL, "See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html"); - terminal.errorPrintln(Verbosity.NORMAL, "for descriptions of what these permissions allow and the associated risks."); - prompt(terminal, batch); } } - private static void prompt(final Terminal terminal, final boolean batch) throws UserException { - if (batch == false) { - terminal.println(Verbosity.NORMAL, ""); - String text = terminal.readText("Continue with installation? [y/N]"); - if (text.equalsIgnoreCase("y") == false) { - throw new UserException(ExitCodes.DATA_ERROR, "installation aborted by user"); - } + private static void prompt(final Terminal terminal) throws UserException { + terminal.println(Verbosity.NORMAL, ""); + String text = terminal.readText("Continue with installation? [y/N]"); + if (text.equalsIgnoreCase("y") == false) { + throw new UserException(ExitCodes.DATA_ERROR, "installation aborted by user"); } } @@ -103,7 +122,7 @@ static String formatPermission(Permission permission) { /** * Extract a unique set of permissions from the plugin's policy file. Each permission is formatted for output to users. */ - static Set getPermissionDescriptions(PluginPolicyInfo pluginPolicyInfo, Path tmpDir) throws IOException { + public static Set getPermissionDescriptions(PluginPolicyInfo pluginPolicyInfo, Path tmpDir) throws IOException { Set allPermissions = new HashSet<>(PolicyUtil.getPolicyPermissions(null, pluginPolicyInfo.policy, tmpDir)); for (URL jar : pluginPolicyInfo.jars) { Set jarPermissions = PolicyUtil.getPolicyPermissions(jar, pluginPolicyInfo.policy, tmpDir); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java new file mode 100644 index 0000000000000..72e28efc4faab --- /dev/null +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginSyncException.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +/** + * Thrown when a problem occurs synchronising plugins. + */ +class PluginSyncException extends Exception { + + PluginSyncException(String message) { + super(message); + } + + PluginSyncException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java new file mode 100644 index 0000000000000..27c1f57fb0bc4 --- /dev/null +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.xcontent.DeprecationHandler; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static java.util.Collections.emptyList; + +/** + * This class models the contents of the {@code elasticsearch-plugins.yml} file. This file specifies all the plugins + * that ought to be installed in an Elasticsearch instance, and where to find them if they are not an official + * Elasticsearch plugin. + */ +public class PluginsConfig { + private List plugins; + private String proxy; + + public PluginsConfig() { + plugins = emptyList(); + proxy = null; + } + + public void setPlugins(List plugins) { + this.plugins = plugins == null ? emptyList() : plugins; + } + + public void setProxy(String proxy) { + this.proxy = proxy; + } + + /** + * Validate this instance. For example: + *

    + *
  • All {@link PluginDescriptor}s must have IDs
    • + *
  • Any proxy must be well-formed.
    • + *
  • Unofficial plugins must have locations
    • + *
+ * + * @param officialPlugins the plugins that can be installed by name only + * @throws PluginSyncException if validation problems are found + */ + public void validate(Set officialPlugins) throws PluginSyncException { + if (this.plugins.stream().anyMatch(each -> each == null || Strings.hasText(each.getId()) == false)) { + throw new RuntimeException("Cannot have null or empty IDs in [elasticsearch-plugins.yml]"); + } + + final Set uniquePluginIds = new HashSet<>(); + for (final PluginDescriptor plugin : plugins) { + if (uniquePluginIds.add(plugin.getId()) == false) { + throw new PluginSyncException("Duplicate plugin ID [" + plugin.getId() + "] found in [elasticsearch-plugins.yml]"); + } + } + + for (PluginDescriptor plugin : this.plugins) { + if (officialPlugins.contains(plugin.getId()) == false && plugin.getLocation() == null) { + throw new PluginSyncException( + "Must specify location for non-official plugin [" + plugin.getId() + "] in [elasticsearch-plugins.yml]" + ); + } + } + + if (this.proxy != null) { + final String[] parts = this.proxy.split(":"); + if (parts.length != 2) { + throw new PluginSyncException("Malformed [proxy], expected [host:port] in [elasticsearch-plugins.yml]"); + } + + if (ProxyUtils.validateProxy(parts[0], parts[1]) == false) { + throw new PluginSyncException("Malformed [proxy], expected [host:port] in [elasticsearch-plugins.yml]"); + } + } + + for (PluginDescriptor p : plugins) { + if (p.getLocation() != null) { + if (Strings.hasText(p.getLocation()) == false) { + throw new PluginSyncException("Empty location for plugin [" + p.getId() + "]"); + } + + try { + // This also accepts Maven coordinates + new URI(p.getLocation()); + } catch (URISyntaxException e) { + throw new PluginSyncException("Malformed location for plugin [" + p.getId() + "]"); + } + } + } + } + + public List getPlugins() { + return plugins; + } + + public String getProxy() { + return proxy; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PluginsConfig that = (PluginsConfig) o; + return plugins.equals(that.plugins) && Objects.equals(proxy, that.proxy); + } + + @Override + public int hashCode() { + return Objects.hash(plugins, proxy); + } + + @Override + public String toString() { + return "PluginsConfig{plugins=" + plugins + ", proxy='" + proxy + "'}"; + } + + /** + * Constructs a {@link PluginsConfig} instance from the config YAML file + * + * @param configPath the config file to load + * @param xContent the XContent type to expect when reading the file + * @return a validated config + */ + static PluginsConfig parseConfig(Path configPath, XContent xContent) throws IOException { + // Normally a parser is declared and built statically in the class, but we'll only + // use this when starting up Elasticsearch, so there's no point keeping one around. + + final ObjectParser descriptorParser = new ObjectParser<>("descriptor parser", PluginDescriptor::new); + descriptorParser.declareString(PluginDescriptor::setId, new ParseField("id")); + descriptorParser.declareStringOrNull(PluginDescriptor::setLocation, new ParseField("location")); + + final ObjectParser parser = new ObjectParser<>("plugins parser", PluginsConfig::new); + parser.declareStringOrNull(PluginsConfig::setProxy, new ParseField("proxy")); + parser.declareObjectArrayOrNull(PluginsConfig::setPlugins, descriptorParser, new ParseField("plugins")); + + final XContentParser yamlXContentParser = xContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + Files.newInputStream(configPath) + ); + + return parser.parse(yamlXContentParser, null); + } + + /** + * Write a config file to disk + * @param xContent the format to use when writing the config + * @param config the config to write + * @param configPath the path to write to + * @throws IOException if anything breaks + */ + static void writeConfig(XContent xContent, PluginsConfig config, Path configPath) throws IOException { + final OutputStream outputStream = Files.newOutputStream(configPath); + final XContentBuilder builder = new XContentBuilder(xContent, outputStream); + + builder.startObject(); + builder.startArray("plugins"); + for (PluginDescriptor p : config.getPlugins()) { + builder.startObject(); + { + builder.field("id", p.getId()); + builder.field("location", p.getLocation()); + } + builder.endObject(); + } + builder.endArray(); + builder.field("proxy", config.getProxy()); + builder.endObject(); + + builder.close(); + outputStream.close(); + } +} diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java index ff79f7acd8db2..c162c27a896fc 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java @@ -20,13 +20,13 @@ * * Only used by the InstallPluginCommand, thus package private here */ -abstract class ProgressInputStream extends FilterInputStream { +public abstract class ProgressInputStream extends FilterInputStream { private final int expectedTotalSize; private int currentPercent; private int count = 0; - ProgressInputStream(InputStream is, int expectedTotalSize) { + public ProgressInputStream(InputStream is, int expectedTotalSize) { super(is); this.expectedTotalSize = expectedTotalSize; this.currentPercent = 0; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java new file mode 100644 index 0000000000000..e2938052d13eb --- /dev/null +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProxyUtils.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.SuppressForbidden; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.Strings; + +import java.net.InetSocketAddress; +import java.net.Proxy; + +/** + * Utilities for working with HTTP proxies. + */ +class ProxyUtils { + /** + * Constructs a proxy from the given string. If {@code null} is passed, then {@code null} will + * be returned, since that is not the same as {@link Proxy#NO_PROXY}. + * + * @param proxy the string to use, in the form "host:port" + * @return a proxy or null + */ + @SuppressForbidden(reason = "Proxy constructor requires a SocketAddress") + static Proxy buildProxy(String proxy) throws UserException { + if (proxy == null) { + return null; + } + + final String[] parts = proxy.split(":"); + if (parts.length != 2) { + throw new UserException(ExitCodes.CONFIG, "Malformed [proxy], expected [host:port]"); + } + + if (validateProxy(parts[0], parts[1]) == false) { + throw new UserException(ExitCodes.CONFIG, "Malformed [proxy], expected [host:port]"); + } + + return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(parts[0], Integer.parseUnsignedInt(parts[1]))); + } + + /** + * Check that the hostname is not empty, and that the port is numeric. + * + * @param hostname the hostname to check. Besides ensuring it is not null or empty, no further validation is + * performed. + * @param port the port to check. Must be composed solely of digits. + * @return whether the arguments describe a potentially valid proxy. + */ + static boolean validateProxy(String hostname, String port) { + return Strings.isNullOrEmpty(hostname) == false && port != null && port.matches("^\\d+$") != false; + } +} diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java index 271cf13d3f461..0d195b61c131c 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java @@ -34,7 +34,7 @@ /** * An action for the plugin CLI to remove plugins from Elasticsearch. */ -class RemovePluginAction { +public class RemovePluginAction { // exit codes for remove /** A plugin cannot be removed because it is extended by another plugin. */ @@ -42,7 +42,7 @@ class RemovePluginAction { private final Terminal terminal; private final Environment env; - private final boolean purge; + private boolean purge; /** * Creates a new action. @@ -51,12 +51,16 @@ class RemovePluginAction { * @param env the environment for the local node * @param purge if true, plugin configuration files will be removed but otherwise preserved */ - RemovePluginAction(Terminal terminal, Environment env, boolean purge) { + public RemovePluginAction(Terminal terminal, Environment env, boolean purge) { this.terminal = terminal; this.env = env; this.purge = purge; } + public void setPurge(boolean purge) { + this.purge = purge; + } + /** * Remove the plugin specified by {@code pluginName}. * @@ -66,7 +70,7 @@ class RemovePluginAction { * @throws UserException if plugin directory does not exist * @throws UserException if the plugin bin directory is not a directory */ - void execute(List plugins) throws IOException, UserException { + public void execute(List plugins) throws IOException, UserException { if (plugins == null || plugins.isEmpty()) { throw new UserException(ExitCodes.USAGE, "At least one plugin ID is required"); } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java index 34bdf55e02a20..0cb0c927f18d4 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java @@ -34,6 +34,8 @@ class RemovePluginCommand extends EnvironmentAwareCommand { @Override protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws Exception { + SyncPluginsAction.ensureNoConfigFile(env); + final List plugins = arguments.values(options).stream().map(PluginDescriptor::new).collect(Collectors.toList()); final RemovePluginAction action = new RemovePluginAction(terminal, env, options.has(purgeOption)); diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java new file mode 100644 index 0000000000000..b5d2a42df2812 --- /dev/null +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java @@ -0,0 +1,314 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.Version; +import org.elasticsearch.cli.ExitCodes; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.PluginInfo; +import org.elasticsearch.plugins.PluginsSynchronizer; +import org.elasticsearch.xcontent.cbor.CborXContent; +import org.elasticsearch.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.net.Proxy; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyMap; + +/** + * This action compares the contents of a configuration files, {@code elasticsearch-plugins.yml}, with the currently + * installed plugins, and ensures that plugins are installed or removed accordingly. + *

+ * This action cannot be called from the command line. It is used exclusively by Elasticsearch on startup, but only + * if the config file exists and the distribution type allows it. + */ +public class SyncPluginsAction implements PluginsSynchronizer { + public static final String ELASTICSEARCH_PLUGINS_YML = "elasticsearch-plugins.yml"; + public static final String ELASTICSEARCH_PLUGINS_YML_CACHE = ".elasticsearch-plugins.yml.cache"; + + private final Terminal terminal; + private final Environment env; + + public SyncPluginsAction(Terminal terminal, Environment env) { + this.terminal = terminal; + this.env = env; + } + + /** + * Ensures that the plugin config file does not exist. + * @param env the environment to check + * @throws UserException if a plugins config file is found. + */ + public static void ensureNoConfigFile(Environment env) throws UserException { + final Path pluginsConfig = env.configFile().resolve("elasticsearch-plugins.yml"); + if (Files.exists(pluginsConfig)) { + throw new UserException( + ExitCodes.USAGE, + "Plugins config [" + + pluginsConfig + + "] exists, which is used by Elasticsearch on startup to ensure the correct plugins " + + "are installed. Instead of using this tool, you need to update this config file and restart Elasticsearch." + ); + } + } + + /** + * Synchronises plugins from the config file to the plugins dir. + * + * @throws Exception if anything goes wrong + */ + @Override + public void execute() throws Exception { + final Path configPath = this.env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML); + final Path previousConfigPath = this.env.pluginsFile().resolve(ELASTICSEARCH_PLUGINS_YML_CACHE); + + if (Files.exists(configPath) == false) { + // The `PluginsManager` will have checked that this file exists before invoking the action. + throw new PluginSyncException("Plugins config does not exist: " + configPath.toAbsolutePath()); + } + + if (Files.exists(env.pluginsFile()) == false) { + throw new PluginSyncException("Plugins directory missing: " + env.pluginsFile()); + } + + // Parse descriptor file + final PluginsConfig pluginsConfig = PluginsConfig.parseConfig(configPath, YamlXContent.yamlXContent); + pluginsConfig.validate(InstallPluginAction.OFFICIAL_PLUGINS); + + // Parse cached descriptor file, if it exists + final Optional cachedPluginsConfig = Files.exists(previousConfigPath) + ? Optional.of(PluginsConfig.parseConfig(previousConfigPath, CborXContent.cborXContent)) + : Optional.empty(); + + final PluginChanges changes = getPluginChanges(pluginsConfig, cachedPluginsConfig); + + if (changes.isEmpty()) { + terminal.println("No plugins to install, remove or upgrade"); + return; + } + + performSync(pluginsConfig, changes); + + // 8. Cached the applied config so that we can diff it on the next run. + PluginsConfig.writeConfig(CborXContent.cborXContent, pluginsConfig, previousConfigPath); + } + + // @VisibleForTesting + PluginChanges getPluginChanges(PluginsConfig pluginsConfig, Optional cachedPluginsConfig) throws PluginSyncException { + final List existingPlugins = getExistingPlugins(this.env); + + final List pluginsThatShouldExist = pluginsConfig.getPlugins(); + final List pluginsThatActuallyExist = existingPlugins.stream() + .map(info -> new PluginDescriptor(info.getName())) + .collect(Collectors.toList()); + final Set existingPluginIds = pluginsThatActuallyExist.stream().map(PluginDescriptor::getId).collect(Collectors.toSet()); + + final List pluginsToInstall = difference(pluginsThatShouldExist, pluginsThatActuallyExist); + final List pluginsToRemove = difference(pluginsThatActuallyExist, pluginsThatShouldExist); + + // Candidates for upgrade are any plugin that already exist and isn't about to be removed. + final List pluginsToMaybeUpgrade = difference(pluginsThatShouldExist, pluginsToRemove).stream() + .filter(each -> existingPluginIds.contains(each.getId())) + .collect(Collectors.toList()); + + final List pluginsToUpgrade = getPluginsToUpgrade(pluginsToMaybeUpgrade, cachedPluginsConfig, existingPlugins); + + return new PluginChanges(pluginsToRemove, pluginsToInstall, pluginsToUpgrade); + } + + private void performSync(PluginsConfig pluginsConfig, PluginChanges changes) throws Exception { + final Proxy proxy = ProxyUtils.buildProxy(pluginsConfig.getProxy()); + + final RemovePluginAction removePluginAction = new RemovePluginAction(terminal, env, true); + final InstallPluginAction installPluginAction = new InstallPluginAction(terminal, env, true); + installPluginAction.setProxy(proxy); + + performSync(installPluginAction, removePluginAction, changes); + } + + // @VisibleForTesting + void performSync(InstallPluginAction installAction, RemovePluginAction removeAction, PluginChanges changes) throws Exception { + logRequiredChanges(changes); + + // Remove any plugins that are not in the config file + if (changes.remove.isEmpty() == false) { + removeAction.setPurge(true); + removeAction.execute(changes.remove); + } + + // Add any plugins that are in the config file but missing from disk + if (changes.install.isEmpty() == false) { + installAction.execute(changes.install); + } + + // Upgrade plugins + if (changes.upgrade.isEmpty() == false) { + removeAction.setPurge(false); + removeAction.execute(changes.upgrade); + installAction.execute(changes.upgrade); + } + } + + private List getPluginsToUpgrade( + List pluginsToMaybeUpgrade, + Optional cachedPluginsConfig, + List existingPlugins + ) { + final Map cachedPluginIdToLocation = cachedPluginsConfig.map( + config -> config.getPlugins().stream().collect(Collectors.toMap(PluginDescriptor::getId, PluginDescriptor::getLocation)) + ).orElse(emptyMap()); + + return pluginsToMaybeUpgrade.stream().filter(eachPlugin -> { + final String eachPluginId = eachPlugin.getId(); + + // If a plugin's location has changed, reinstall + if (Objects.equals(eachPlugin.getLocation(), cachedPluginIdToLocation.get(eachPluginId)) == false) { + this.terminal.println( + Terminal.Verbosity.VERBOSE, + String.format( + Locale.ROOT, + "Location for plugin [%s] has changed from [%s] to [%s], reinstalling", + eachPluginId, + cachedPluginIdToLocation.get(eachPluginId), + eachPlugin.getLocation() + ) + ); + return true; + } + + // Official plugins must be upgraded when an Elasticsearch node is upgraded. + if (InstallPluginAction.OFFICIAL_PLUGINS.contains(eachPluginId)) { + // Find the currently installed plugin and check whether the version is lower than + // the current node's version. + final PluginInfo info = existingPlugins.stream() + .filter(each -> each.getName().equals(eachPluginId)) + .findFirst() + .orElseThrow(() -> { + // It should be literally impossible for us not to find a matching existing plugin. We derive + // the list of existing plugin IDs from the list of installed plugins. + throw new RuntimeException("Couldn't find a PluginInfo for [" + eachPluginId + "], which should be impossible"); + }); + + if (info.getElasticsearchVersion().before(Version.CURRENT)) { + this.terminal.println( + Terminal.Verbosity.VERBOSE, + String.format( + Locale.ROOT, + "Official plugin [%s] is out-of-date (%s versus %s), upgrading", + eachPluginId, + info.getElasticsearchVersion(), + Version.CURRENT + ) + ); + return true; + } + return false; + } + + // Else don't upgrade. + return false; + }).collect(Collectors.toList()); + } + + private List getExistingPlugins(Environment env) throws PluginSyncException { + final List plugins = new ArrayList<>(); + + try { + try (DirectoryStream paths = Files.newDirectoryStream(env.pluginsFile())) { + for (Path pluginPath : paths) { + String filename = pluginPath.getFileName().toString(); + if (filename.startsWith(".")) { + continue; + } + + PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(pluginPath)); + plugins.add(info); + + // Check for a version mismatch, unless it's an official plugin since we can upgrade them. + if (InstallPluginAction.OFFICIAL_PLUGINS.contains(info.getName()) + && info.getElasticsearchVersion().equals(Version.CURRENT) == false) { + this.terminal.errorPrintln( + String.format( + Locale.ROOT, + "WARNING: plugin [%s] was built for Elasticsearch version %s but version %s is required", + info.getName(), + info.getElasticsearchVersion(), + Version.CURRENT + ) + ); + } + } + } + } catch (IOException e) { + throw new PluginSyncException("Failed to list existing plugins", e); + } + + return plugins; + } + + /** + * Returns a list of all elements in {@code left} that are not present in {@code right}. + *

+ * Comparisons are based solely using {@link PluginDescriptor#getId()}. + * + * @param left the items that may be retained + * @param right the items that may be removed + * @return a list of the remaining elements + */ + private static List difference(List left, List right) { + return left.stream().filter(eachDescriptor -> { + final String id = eachDescriptor.getId(); + return right.stream().anyMatch(p -> p.getId().equals(id)) == false; + }).collect(Collectors.toList()); + } + + private void logRequiredChanges(PluginChanges changes) { + final BiConsumer> printSummary = (action, plugins) -> { + if (plugins.isEmpty() == false) { + List pluginIds = plugins.stream().map(PluginDescriptor::getId).collect(Collectors.toList()); + this.terminal.errorPrintln(String.format(Locale.ROOT, "Plugins to be %s: %s", action, pluginIds)); + } + }; + + printSummary.accept("removed", changes.remove); + printSummary.accept("installed", changes.install); + printSummary.accept("upgraded", changes.upgrade); + } + + // @VisibleForTesting + static class PluginChanges { + final List remove; + final List install; + final List upgrade; + + PluginChanges(List remove, List install, List upgrade) { + this.remove = Objects.requireNonNull(remove); + this.install = Objects.requireNonNull(install); + this.upgrade = Objects.requireNonNull(upgrade); + } + + boolean isEmpty() { + return remove.isEmpty() && install.isEmpty() && upgrade.isEmpty(); + } + } +} diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index 9c5399fe0b42b..22b435c5384e3 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -223,7 +223,7 @@ static Path writeZip(Path structure, String prefix) throws IOException { Path zip = createTempDir().resolve(structure.getFileName() + ".zip"); try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { forEachFileRecursively(structure, (file, attrs) -> { - String target = (prefix == null ? "" : prefix + "/") + structure.relativize(file).toString(); + String target = (prefix == null ? "" : prefix + "/") + structure.relativize(file); stream.putNextEntry(new ZipEntry(target)); Files.copy(file, stream); }); @@ -417,20 +417,20 @@ public void testMultipleWorks() throws Exception { public void testDuplicateInstall() throws Exception { PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); final UserException e = expectThrows(UserException.class, () -> installPlugins(Arrays.asList(pluginZip, pluginZip), env.v1())); - assertThat(e, hasToString(containsString("duplicate plugin id [" + pluginZip.getId() + "]"))); + assertThat(e.getMessage(), equalTo("duplicate plugin id [" + pluginZip.getId() + "]")); } public void testTransaction() throws Exception { PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); PluginDescriptor nonexistentPluginZip = new PluginDescriptor( pluginZip.getId() + "-does-not-exist", - pluginZip.getUrl() + "-does-not-exist" + pluginZip.getLocation() + "-does-not-exist" ); final FileNotFoundException e = expectThrows( FileNotFoundException.class, () -> installPlugins(Arrays.asList(pluginZip, nonexistentPluginZip), env.v1()) ); - assertThat(e, hasToString(containsString("does-not-exist"))); + assertThat(e.getMessage(), containsString("does-not-exist")); final Path fakeInstallPath = env.v2().pluginsFile().resolve("fake"); // fake should have been removed when the file not found exception occurred assertFalse(Files.exists(fakeInstallPath)); @@ -447,13 +447,13 @@ public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception { "found file [%s] from a failed attempt to remove the plugin [failed]; execute [elasticsearch-plugin remove failed]", removing ); - assertThat(e, hasToString(containsString(expected))); + assertThat(e.getMessage(), containsString(expected)); } public void testSpaceInUrl() throws Exception { PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); Path pluginZipWithSpaces = createTempFile("foo bar", ".zip"); - try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip.getUrl()))) { + try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip.getLocation()))) { Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING); } PluginDescriptor modifiedPlugin = new PluginDescriptor("fake", pluginZipWithSpaces.toUri().toURL().toString()); @@ -465,7 +465,7 @@ public void testMalformedUrlNotMaven() { // has two colons, so it appears similar to maven coordinates PluginDescriptor plugin = new PluginDescriptor("fake", "://host:1234"); MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin(plugin)); - assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); + assertThat(e.getMessage(), containsString("no protocol")); } public void testFileNotMaven() { @@ -475,13 +475,13 @@ public void testFileNotMaven() { // has two colons, so it appears similar to maven coordinates () -> installPlugin("file:" + dir) ); - assertFalse(e.getMessage(), e.getMessage().contains("maven.org")); - assertTrue(e.getMessage(), e.getMessage().contains(dir)); + assertThat(e.getMessage(), not(containsString("maven.org"))); + assertThat(e.getMessage(), containsString(dir)); } public void testUnknownPlugin() { UserException e = expectThrows(UserException.class, () -> installPlugin("foo")); - assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo")); + assertThat(e.getMessage(), containsString("Unknown plugin foo")); } public void testPluginsDirReadOnly() throws Exception { @@ -490,7 +490,7 @@ public void testPluginsDirReadOnly() throws Exception { pluginsAttrs.setPermissions(new HashSet<>()); PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains(env.v2().pluginsFile().toString())); + assertThat(e.getMessage(), containsString(env.v2().pluginsFile().toString())); } assertInstallCleaned(env.v2()); } @@ -498,7 +498,7 @@ public void testPluginsDirReadOnly() throws Exception { public void testBuiltinModule() throws Exception { PluginDescriptor pluginZip = createPluginZip("lang-painless", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("is a system module")); + assertThat(e.getMessage(), containsString("is a system module")); assertInstallCleaned(env.v2()); } @@ -508,7 +508,7 @@ public void testBuiltinXpackModule() throws Exception { // whose descriptor contains the name "x-pack". pluginZip.setId("not-x-pack"); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("is a system module")); + assertThat(e.getMessage(), containsString("is a system module")); assertInstallCleaned(env.v2()); } @@ -519,7 +519,7 @@ public void testJarHell() throws Exception { writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin"); PluginDescriptor pluginZip = createPluginZip("fake", pluginDirectory); // adds plugin.jar with FakePlugin IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip, env.v1(), defaultAction)); - assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); + assertThat(e.getMessage(), containsString("jar hell")); assertInstallCleaned(env.v2()); } @@ -539,7 +539,7 @@ public void testExistingPlugin() throws Exception { PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); installPlugin(pluginZip); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("already exists")); + assertThat(e.getMessage(), containsString("already exists")); assertInstallCleaned(env.v2()); } @@ -557,7 +557,7 @@ public void testBinNotDir() throws Exception { Files.createFile(binDir); PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); + assertThat(e.getMessage(), containsString("not a directory")); assertInstallCleaned(env.v2()); } @@ -567,7 +567,7 @@ public void testBinContainsDir() throws Exception { Files.createFile(dirInBinDir.resolve("somescript")); PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin")); + assertThat(e.getMessage(), containsString("Directories not allowed in bin dir for plugin")); assertInstallCleaned(env.v2()); } @@ -577,7 +577,7 @@ public void testBinConflict() throws Exception { Files.createFile(binDir.resolve("somescript")); PluginDescriptor pluginZip = createPluginZip("elasticsearch", pluginDir); FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains(env.v2().binFile().resolve("elasticsearch").toString())); + assertThat(e.getMessage(), containsString(env.v2().binFile().resolve("elasticsearch").toString())); assertInstallCleaned(env.v2()); } @@ -689,7 +689,7 @@ public void testConfigNotDir() throws Exception { Files.createFile(configDir); PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); + assertThat(e.getMessage(), containsString("not a directory")); assertInstallCleaned(env.v2()); } @@ -699,7 +699,7 @@ public void testConfigContainsDir() throws Exception { Files.createFile(dirInConfigDir.resolve("myconfig.yml")); PluginDescriptor pluginZip = createPluginZip("fake", pluginDir); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin")); + assertThat(e.getMessage(), containsString("Directories not allowed in config dir for plugin")); assertInstallCleaned(env.v2()); } @@ -707,7 +707,7 @@ public void testMissingDescriptor() throws Exception { Files.createFile(pluginDir.resolve("fake.yml")); String pluginZip = writeZip(pluginDir, null).toUri().toURL().toString(); NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties")); + assertThat(e.getMessage(), containsString("plugin-descriptor.properties")); assertInstallCleaned(env.v2()); } @@ -726,18 +726,13 @@ public void testZipRelativeOutsideEntryName() throws Exception { } String pluginZip = zip.toUri().toURL().toString(); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory")); + assertThat(e.getMessage(), containsString("resolving outside of plugin directory")); assertInstallCleaned(env.v2()); } public void testOfficialPluginsHelpSortedAndMissingObviouslyWrongPlugins() throws Exception { MockTerminal terminal = new MockTerminal(); - new InstallPluginCommand() { - @Override - protected boolean addShutdownHook() { - return false; - } - }.main(new String[] { "--help" }, terminal); + new MockInstallPluginCommand().main(new String[] { "--help" }, terminal); try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) { String line = reader.readLine(); @@ -781,7 +776,7 @@ Build.Flavor buildFlavor() { } }; final T exception = expectThrows(clazz, () -> flavorAction.execute(Collections.singletonList(new PluginDescriptor("x-pack")))); - assertThat(exception, hasToString(containsString(expectedMessage))); + assertThat(exception.getMessage(), containsString(expectedMessage)); } public void testInstallMisspelledOfficialPlugins() { @@ -833,6 +828,18 @@ public void testPluginAlreadyInstalled() throws Exception { ); } + /** + * Check that if the installer action finds a mismatch between what it expects a plugin's ID to be and what + * the ID actually is from the plugin's properties, then the installation fails. + */ + public void testPluginHasDifferentNameThatDescriptor() throws Exception { + PluginDescriptor descriptor = createPluginZip("fake", pluginDir); + PluginDescriptor modifiedDescriptor = new PluginDescriptor("other-fake", descriptor.getLocation()); + + final UserException e = expectThrows(UserException.class, () -> installPlugin(modifiedDescriptor)); + assertThat(e.getMessage(), equalTo("Expected downloaded plugin to have ID [other-fake] but found [fake]")); + } + private void installPlugin(boolean isBatch, String... additionalProperties) throws Exception { // if batch is enabled, we also want to add a security policy if (isBatch) { @@ -844,10 +851,36 @@ private void installPlugin(boolean isBatch, String... additionalProperties) thro skipJarHellAction.execute(Collections.singletonList(pluginZip)); } + private void assertInstallPluginFromUrl(final String pluginId, final String url, final String stagingHash, boolean isSnapshot) + throws Exception { + assertInstallPluginFromUrl(pluginId, null, url, stagingHash, isSnapshot); + } + + private void assertInstallPluginFromUrl( + final String pluginId, + final String pluginUrl, + final String url, + final String stagingHash, + boolean isSnapshot + ) throws Exception { + final MessageDigest digest = MessageDigest.getInstance("SHA-512"); + assertInstallPluginFromUrl( + pluginId, + pluginUrl, + url, + stagingHash, + isSnapshot, + ".sha512", + checksumAndFilename(digest, url), + newSecretKey(), + this::signature + ); + } + @SuppressForbidden(reason = "Paths.get() is OK in this context") void assertInstallPluginFromUrl( final String pluginId, - final String name, + final String pluginUrl, final String url, final String stagingHash, final boolean isSnapshot, @@ -856,8 +889,8 @@ void assertInstallPluginFromUrl( final PGPSecretKey secretKey, final BiFunction signature ) throws Exception { - PluginDescriptor pluginZip = createPlugin(name, pluginDir); - Path pluginZipPath = Paths.get(URI.create(pluginZip.getUrl())); + PluginDescriptor pluginZip = createPlugin(pluginId, pluginDir); + Path pluginZipPath = Paths.get(URI.create(pluginZip.getLocation())); InstallPluginAction action = new InstallPluginAction(terminal, env.v2(), false) { @Override Path downloadZip(String urlString, Path tmpDir) throws IOException { @@ -870,7 +903,7 @@ Path downloadZip(String urlString, Path tmpDir) throws IOException { @Override URL openUrl(String urlString) throws IOException { if ((url + shaExtension).equals(urlString)) { - // calc sha an return file URL to it + // calc sha and return file URL to it Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension); byte[] zipbytes = Files.readAllBytes(pluginZipPath); String checksum = shaCalculator.apply(zipbytes); @@ -888,7 +921,7 @@ URL openUrl(String urlString) throws IOException { @Override void verifySignature(Path zip, String urlString) throws IOException, PGPException { - if (InstallPluginAction.OFFICIAL_PLUGINS.contains(name)) { + if (InstallPluginAction.OFFICIAL_PLUGINS.contains(pluginId)) { super.verifySignature(zip, urlString); } else { throw new UnsupportedOperationException("verify signature should not be called for unofficial plugins"); @@ -938,36 +971,15 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Pat // no jarhell check } }; - installPlugin(new PluginDescriptor(name, pluginId), env.v1(), action); - assertPlugin(name, pluginDir, env.v2()); - } - - public void assertInstallPluginFromUrl( - final String pluginId, - final String name, - final String url, - final String stagingHash, - boolean isSnapshot - ) throws Exception { - final MessageDigest digest = MessageDigest.getInstance("SHA-512"); - assertInstallPluginFromUrl( - pluginId, - name, - url, - stagingHash, - isSnapshot, - ".sha512", - checksumAndFilename(digest, url), - newSecretKey(), - this::signature - ); + installPlugin(new PluginDescriptor(pluginId, pluginUrl), env.v1(), action); + assertPlugin(pluginId, pluginDir, env.v2()); } public void testOfficialPlugin() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + assertInstallPluginFromUrl("analysis-icu", url, null, false); } public void testOfficialPluginSnapshot() throws Exception { @@ -977,7 +989,7 @@ public void testOfficialPluginSnapshot() throws Exception { Version.CURRENT, Build.CURRENT.getQualifiedVersion() ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); + assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { @@ -987,15 +999,15 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { Version.CURRENT, Build.CURRENT.getQualifiedVersion() ); - // attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception + // attempting to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception final UserException e = expectThrows( UserException.class, () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true) ); assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); assertThat( - e, - hasToString(containsString("attempted to install release build of official plugin on snapshot build of Elasticsearch")) + e.getMessage(), + containsString("attempted to install release build of official plugin on snapshot build of Elasticsearch") ); } @@ -1005,7 +1017,7 @@ public void testOfficialPluginStaging() throws Exception { + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", url, "abc123", false); } public void testOfficialPlatformPlugin() throws Exception { @@ -1014,7 +1026,7 @@ public void testOfficialPlatformPlugin() throws Exception { + "-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + assertInstallPluginFromUrl("analysis-icu", url, null, false); } public void testOfficialPlatformPluginSnapshot() throws Exception { @@ -1025,7 +1037,7 @@ public void testOfficialPlatformPluginSnapshot() throws Exception { Platforms.PLATFORM_NAME, Build.CURRENT.getQualifiedVersion() ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); + assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } public void testOfficialPlatformPluginStaging() throws Exception { @@ -1036,23 +1048,23 @@ public void testOfficialPlatformPluginStaging() throws Exception { + "-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", url, "abc123", false); } public void testMavenPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("myplugin", "mygroup:myplugin:1.0.0", url, null, false); } public void testMavenPlatformPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("myplugin", "mygroup:myplugin:1.0.0", url, null, false); } public void testMavenSha1Backcompat() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest), null, (b, p) -> null); + assertInstallPluginFromUrl("myplugin", "mygroup:myplugin:1.0.0", url, null, false, ".sha1", checksum(digest), null, (b, p) -> null); assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); } @@ -1060,8 +1072,8 @@ public void testMavenChecksumWithoutFilename() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-512"); assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", "myplugin", + "mygroup:myplugin:1.0.0", url, null, false, @@ -1079,17 +1091,7 @@ public void testOfficialChecksumWithoutFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", null, url, null, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertThat(e.getMessage(), startsWith("Invalid checksum file")); @@ -1102,20 +1104,10 @@ public void testOfficialShaMissing() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-1"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha1", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", null, url, null, false, ".sha1", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); + assertThat(e.getMessage(), equalTo("Plugin checksum missing: " + url + ".sha512")); } public void testMavenShaMissing() { @@ -1123,8 +1115,8 @@ public void testMavenShaMissing() { UserException e = expectThrows( UserException.class, () -> assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", "myplugin", + "mygroup:myplugin:1.0.0", url, null, false, @@ -1135,7 +1127,7 @@ public void testMavenShaMissing() { ) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); + assertThat(e.getMessage(), equalTo("Plugin checksum missing: " + url + ".sha1")); } public void testInvalidShaFileMissingFilename() throws Exception { @@ -1145,20 +1137,10 @@ public void testInvalidShaFileMissingFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", null, url, null, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); + assertThat(e.getMessage(), containsString("Invalid checksum file")); } public void testInvalidShaFileMismatchFilename() throws Exception { @@ -1170,7 +1152,7 @@ public void testInvalidShaFileMismatchFilename() throws Exception { UserException.class, () -> assertInstallPluginFromUrl( "analysis-icu", - "analysis-icu", + null, url, null, false, @@ -1193,7 +1175,7 @@ public void testInvalidShaFileContainingExtraLine() throws Exception { UserException.class, () -> assertInstallPluginFromUrl( "analysis-icu", - "analysis-icu", + null, url, null, false, @@ -1204,7 +1186,7 @@ public void testInvalidShaFileContainingExtraLine() throws Exception { ) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); + assertThat(e.getMessage(), containsString("Invalid checksum file")); } public void testSha512Mismatch() { @@ -1215,7 +1197,7 @@ public void testSha512Mismatch() { UserException.class, () -> assertInstallPluginFromUrl( "analysis-icu", - "analysis-icu", + null, url, null, false, @@ -1226,7 +1208,7 @@ public void testSha512Mismatch() { ) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar")); + assertThat(e.getMessage(), containsString("SHA-512 mismatch, expected foobar")); } public void testSha1Mismatch() { @@ -1234,8 +1216,8 @@ public void testSha1Mismatch() { UserException e = expectThrows( UserException.class, () -> assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", "myplugin", + "mygroup:myplugin:1.0.0", url, null, false, @@ -1246,7 +1228,7 @@ public void testSha1Mismatch() { ) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); - assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar")); + assertThat(e.getMessage(), containsString("SHA-1 mismatch, expected foobar")); } public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception { @@ -1271,7 +1253,7 @@ public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception { IllegalStateException.class, () -> assertInstallPluginFromUrl( icu, - icu, + null, url, null, false, @@ -1281,7 +1263,7 @@ public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception { signature ) ); - assertThat(e, hasToString(containsString("key id [" + actualID + "] does not match expected key id [" + expectedID + "]"))); + assertThat(e.getMessage(), containsString("key id [" + actualID + "] does not match expected key id [" + expectedID + "]")); } public void testFailedSignatureVerification() throws Exception { @@ -1306,7 +1288,7 @@ public void testFailedSignatureVerification() throws Exception { IllegalStateException.class, () -> assertInstallPluginFromUrl( icu, - icu, + null, url, null, false, @@ -1316,7 +1298,7 @@ public void testFailedSignatureVerification() throws Exception { signature ) ); - assertThat(e, hasToString(equalTo("java.lang.IllegalStateException: signature verification for [" + url + "] failed"))); + assertThat(e.getMessage(), containsString("signature verification for [" + url + "] failed")); } public PGPSecretKey newSecretKey() throws NoSuchAlgorithmException, PGPException { @@ -1389,7 +1371,7 @@ private void assertPolicyConfirmation(Tuple env, PluginDescri // default answer, does not install terminal.addTextInput(""); UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertEquals("installation aborted by user", e.getMessage()); + assertThat(e.getMessage(), containsString("installation aborted by user")); assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning)); try (Stream fileStream = Files.list(env.v2().pluginsFile())) { @@ -1403,7 +1385,7 @@ private void assertPolicyConfirmation(Tuple env, PluginDescri } terminal.addTextInput("n"); e = expectThrows(UserException.class, () -> installPlugin(pluginZip)); - assertEquals("installation aborted by user", e.getMessage()); + assertThat(e.getMessage(), containsString("installation aborted by user")); assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning)); try (Stream fileStream = Files.list(env.v2().pluginsFile())) { assertThat(fileStream.collect(Collectors.toList()), empty()); @@ -1433,7 +1415,7 @@ public void testPluginWithNativeController() throws Exception { PluginDescriptor pluginZip = createPluginZip("fake", pluginDir, "has.native.controller", "true"); final IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip)); - assertThat(e, hasToString(containsString("plugins can not have native controllers"))); + assertThat(e.getMessage(), containsString("plugins can not have native controllers")); } public void testMultipleJars() throws Exception { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java new file mode 100644 index 0000000000000..ab6b5d083dd51 --- /dev/null +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockInstallPluginCommand.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.cli.UserException; +import org.elasticsearch.env.Environment; + +import java.util.Map; + +public class MockInstallPluginCommand extends InstallPluginCommand { + private final Environment env; + + public MockInstallPluginCommand(Environment env) { + this.env = env; + } + + public MockInstallPluginCommand() { + this.env = null; + } + + @Override + protected Environment createEnv(Map settings) throws UserException { + return this.env != null ? this.env : super.createEnv(settings); + } + + @Override + protected boolean addShutdownHook() { + return false; + } +} diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java new file mode 100644 index 0000000000000..ce3555275652c --- /dev/null +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/MockRemovePluginCommand.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.env.Environment; + +import java.util.Map; + +public class MockRemovePluginCommand extends RemovePluginCommand { + final Environment env; + + public MockRemovePluginCommand(final Environment env) { + this.env = env; + } + + @Override + protected Environment createEnv(Map settings) { + return env; + } +} diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java new file mode 100644 index 0000000000000..f20b95b94bbf8 --- /dev/null +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyMatcher.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.cli.SuppressForbidden; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +import java.net.InetSocketAddress; +import java.net.Proxy; + +class ProxyMatcher extends TypeSafeMatcher { + private final Proxy.Type type; + private final String hostname; + private final int port; + + public static ProxyMatcher matchesProxy(Proxy.Type type, String hostname, int port) { + return new ProxyMatcher(type, hostname, port); + } + + public static ProxyMatcher matchesProxy(Proxy.Type type) { + return new ProxyMatcher(type, null, -1); + } + + ProxyMatcher(Proxy.Type type, String hostname, int port) { + this.type = type; + this.hostname = hostname; + this.port = port; + } + + @Override + @SuppressForbidden(reason = "Proxy constructor uses InetSocketAddress") + protected boolean matchesSafely(Proxy proxy) { + if (proxy.type() != this.type) { + return false; + } + + if (hostname == null) { + return true; + } + + InetSocketAddress address = (InetSocketAddress) proxy.address(); + + return this.hostname.equals(address.getHostName()) && this.port == address.getPort(); + } + + @Override + public void describeTo(Description description) { + description.appendText("a proxy instance of type [" + type + "] pointing at [" + hostname + ":" + port + "]"); + } +} diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java new file mode 100644 index 0000000000000..a05289362374e --- /dev/null +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ProxyUtilsTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.cli; + +import org.elasticsearch.cli.UserException; +import org.elasticsearch.test.ESTestCase; + +import java.net.Proxy.Type; +import java.util.stream.Stream; + +import static org.elasticsearch.plugins.cli.ProxyMatcher.matchesProxy; +import static org.elasticsearch.plugins.cli.ProxyUtils.buildProxy; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class ProxyUtilsTests extends ESTestCase { + /** + * Check that building a proxy with just a hostname and port succeeds. + */ + public void testBuildProxy_withHostPort() throws Exception { + assertThat(buildProxy("host:1234"), matchesProxy(Type.HTTP, "host", 1234)); + } + + /** + * Check that building a proxy with a null value succeeds, returning a pass-through (direct) proxy. + */ + public void testBuildProxy_withNullValue() throws Exception { + assertThat(buildProxy(null), is(nullValue())); + } + + /** + * Check that building a proxy with a missing host is rejected. + */ + public void testBuildProxy_withMissingHost() { + UserException e = expectThrows(UserException.class, () -> buildProxy(":1234")); + assertThat(e.getMessage(), equalTo("Malformed [proxy], expected [host:port]")); + } + + /** + * Check that building a proxy with a missing or invalid port is rejected. + */ + public void testBuildProxy_withInvalidPort() { + Stream.of("host:", "host.domain:-1", "host.domain:$PORT", "host.domain:{{port}}", "host.domain").forEach(testCase -> { + UserException e = expectThrows(UserException.class, () -> buildProxy(testCase)); + assertThat(e.getMessage(), equalTo("Malformed [proxy], expected [host:port]")); + }); + } +} diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java index 4f03d5198842c..2e00cd7670901 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java @@ -28,7 +28,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -37,7 +36,6 @@ import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; @LuceneTestCase.SuppressFileSystems("*") public class RemovePluginActionTests extends ESTestCase { @@ -45,19 +43,6 @@ public class RemovePluginActionTests extends ESTestCase { private Path home; private Environment env; - static class MockRemovePluginCommand extends RemovePluginCommand { - final Environment env; - - private MockRemovePluginCommand(final Environment env) { - this.env = env; - } - - @Override - protected Environment createEnv(Map settings) throws UserException { - return env; - } - } - @Override @Before public void setUp() throws Exception { @@ -122,7 +107,7 @@ static void assertRemoveCleaned(Environment env) throws IOException { public void testMissing() throws Exception { UserException e = expectThrows(UserException.class, () -> removePlugin("dne", home, randomBoolean())); - assertTrue(e.getMessage(), e.getMessage().contains("plugin [dne] not found")); + assertThat(e.getMessage(), containsString("plugin [dne] not found")); assertRemoveCleaned(env); } @@ -184,7 +169,7 @@ public void testBinNotDir() throws Exception { createPlugin("fake"); Files.createFile(env.binFile().resolve("fake")); UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, randomBoolean())); - assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); + assertThat(e.getMessage(), containsString("not a directory")); assertTrue(Files.exists(env.pluginsFile().resolve("fake"))); // did not remove assertTrue(Files.exists(env.binFile().resolve("fake"))); assertRemoveCleaned(env); @@ -226,7 +211,7 @@ public void testPurgePluginDoesNotExist() throws Exception { public void testPurgeNothingExists() throws Exception { final UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, true)); - assertThat(e, hasToString(containsString("plugin [fake] not found"))); + assertThat(e.getMessage(), containsString("plugin [fake] not found")); } public void testPurgeOnlyMarkerFileExists() throws Exception { @@ -278,7 +263,7 @@ public void testMissingPluginName() { e = expectThrows(UserException.class, () -> removePlugin(emptyList(), home, randomBoolean())); assertEquals(ExitCodes.USAGE, e.exitCode); - assertEquals("At least one plugin ID is required", e.getMessage()); + assertThat(e.getMessage(), equalTo("At least one plugin ID is required")); } public void testRemoveWhenRemovingMarker() throws Exception { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java new file mode 100644 index 0000000000000..db3f9223f058d --- /dev/null +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -0,0 +1,299 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.plugins.cli; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.plugins.PluginTestUtil; +import org.elasticsearch.plugins.cli.SyncPluginsAction.PluginChanges; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +@LuceneTestCase.SuppressFileSystems("*") +public class SyncPluginsActionTests extends ESTestCase { + private Environment env; + private SyncPluginsAction action; + private PluginsConfig config; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + Path home = createTempDir(); + Settings settings = Settings.builder().put("path.home", home).build(); + env = TestEnvironment.newEnvironment(settings); + Files.createDirectories(env.binFile()); + Files.createFile(env.binFile().resolve("elasticsearch")); + Files.createDirectories(env.configFile()); + Files.createDirectories(env.pluginsFile()); + + action = new SyncPluginsAction(new MockTerminal(), env); + config = new PluginsConfig(); + } + + /** + * Check that when we ensure a plugins config file doesn't exist, and it really doesn't exist, + * then no exception is thrown. + */ + public void test_ensureNoConfigFile_withoutConfig_doesNothing() throws Exception { + SyncPluginsAction.ensureNoConfigFile(env); + } + + /** + * Check that when we ensure a plugins config file doesn't exist, but a file does exist, + * then an exception is thrown. + */ + public void test_ensureNoConfigFile_withConfig_throwsException() throws Exception { + Files.createFile(env.configFile().resolve("elasticsearch-plugins.yml")); + final UserException e = expectThrows(UserException.class, () -> SyncPluginsAction.ensureNoConfigFile(env)); + + assertThat(e.getMessage(), Matchers.matchesPattern("^Plugins config \\[.*] exists.*$")); + } + + /** + * Check that when there are no plugins to install, and no plugins already installed, then we + * calculate that no changes are required. + */ + public void test_getPluginChanges_withNoChanges_returnsNoChanges() throws PluginSyncException { + final SyncPluginsAction.PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); + + assertThat(pluginChanges.isEmpty(), is(true)); + } + + /** + * Check that when there are no plugins in the config file, and a plugin is already installed, then we + * calculate that the plugin needs to be removed. + */ + public void test_getPluginChanges_withExtraPluginOnDisk_returnsPluginToRemove() throws Exception { + createPlugin("my-plugin"); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); + + assertThat(pluginChanges.isEmpty(), is(false)); + assertThat(pluginChanges.install, empty()); + assertThat(pluginChanges.remove, hasSize(1)); + assertThat(pluginChanges.upgrade, empty()); + assertThat(pluginChanges.remove.get(0).getId(), equalTo("my-plugin")); + } + + /** + * Check that when there is a plugin in the config file, and no plugins already installed, then we + * calculate that the plugin needs to be installed. + */ + public void test_getPluginChanges_withPluginToInstall_returnsPluginToInstall() throws Exception { + config.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); + + assertThat(pluginChanges.isEmpty(), is(false)); + assertThat(pluginChanges.install, hasSize(1)); + assertThat(pluginChanges.remove, empty()); + assertThat(pluginChanges.upgrade, empty()); + assertThat(pluginChanges.install.get(0).getId(), equalTo("my-plugin")); + } + + /** + * Check that when there is an unofficial plugin in the config file, and that plugin is already installed + * but needs to be upgraded due to the Elasticsearch version, then we calculate that no changes are required, + * since we can't automatically upgrade it. + */ + public void test_getPluginChanges_withPluginToUpgrade_returnsNoChanges() throws Exception { + createPlugin("my-plugin", Version.CURRENT.previousMajor()); + config.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); + + assertThat(pluginChanges.isEmpty(), is(true)); + } + + /** + * Check that when there is an official plugin in the config file, and that plugin is already installed + * but needs to be upgraded, then we calculate that the plugin needs to be upgraded. + */ + public void test_getPluginChanges_withOfficialPluginToUpgrade_returnsPluginToUpgrade() throws Exception { + createPlugin("analysis-icu", Version.CURRENT.previousMajor()); + config.setPlugins(Collections.singletonList(new PluginDescriptor("analysis-icu"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); + + assertThat(pluginChanges.isEmpty(), is(false)); + assertThat(pluginChanges.install, empty()); + assertThat(pluginChanges.remove, empty()); + assertThat(pluginChanges.upgrade, hasSize(1)); + assertThat(pluginChanges.upgrade.get(0).getId(), equalTo("analysis-icu")); + } + + /** + * Check that if an unofficial plugins' location has not changed in the cached config, then we + * calculate that the plugin does not need to be upgraded. + */ + public void test_getPluginChanges_withCachedConfigAndNoChanges_returnsNoChanges() throws Exception { + createPlugin("my-plugin"); + config.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin", "file://plugin.zip"))); + + final PluginsConfig cachedConfig = new PluginsConfig(); + cachedConfig.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin", "file://plugin.zip"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.of(cachedConfig)); + + assertThat(pluginChanges.isEmpty(), is(true)); + } + + /** + * Check that if an unofficial plugins' location has changed, then we calculate that the plugin + * needs to be upgraded. + */ + public void test_getPluginChanges_withCachedConfigAndChangedLocation_returnsPluginToUpgrade() throws Exception { + createPlugin("my-plugin"); + config.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin", "file:///after.zip"))); + + final PluginsConfig cachedConfig = new PluginsConfig(); + cachedConfig.setPlugins(Collections.singletonList(new PluginDescriptor("my-plugin", "file://before.zip"))); + + final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.of(cachedConfig)); + + assertThat(pluginChanges.isEmpty(), is(false)); + assertThat(pluginChanges.install, empty()); + assertThat(pluginChanges.remove, empty()); + assertThat(pluginChanges.upgrade, hasSize(1)); + assertThat(pluginChanges.upgrade.get(0).getId(), equalTo("my-plugin")); + } + + /** + * Check that if there are no changes to apply, then the install and remove actions are not used. + * This is a redundant test, really, because the sync action exits early if there are no + * changes. + */ + public void test_performSync_withNoChanges_doesNothing() throws Exception { + final InstallPluginAction installAction = mock(InstallPluginAction.class); + final RemovePluginAction removeAction = mock(RemovePluginAction.class); + + action.performSync(installAction, removeAction, new PluginChanges(emptyList(), emptyList(), emptyList())); + + verify(installAction, never()).execute(anyList()); + verify(removeAction, never()).execute(anyList()); + } + + /** + * Check that if there are plugins to remove, then the remove action is used. + */ + public void test_performSync_withPluginsToRemove_callsRemoveAction() throws Exception { + final InstallPluginAction installAction = mock(InstallPluginAction.class); + final RemovePluginAction removeAction = mock(RemovePluginAction.class); + final List pluginDescriptors = Arrays.asList(new PluginDescriptor("plugin1"), new PluginDescriptor("plugin2")); + + action.performSync(installAction, removeAction, new PluginChanges(pluginDescriptors, emptyList(), emptyList())); + + verify(installAction, never()).execute(anyList()); + verify(removeAction).setPurge(true); + verify(removeAction).execute(pluginDescriptors); + } + + /** + * Check that if there are plugins to install, then the install action is used. + */ + public void test_performSync_withPluginsToInstall_callsInstallAction() throws Exception { + final InstallPluginAction installAction = mock(InstallPluginAction.class); + final RemovePluginAction removeAction = mock(RemovePluginAction.class); + final List pluginDescriptors = Arrays.asList(new PluginDescriptor("plugin1"), new PluginDescriptor("plugin2")); + + action.performSync(installAction, removeAction, new PluginChanges(emptyList(), pluginDescriptors, emptyList())); + + verify(installAction).execute(pluginDescriptors); + verify(removeAction, never()).execute(anyList()); + } + + /** + * Check that if there are plugins to upgrade, then both the install and remove actions are used. + */ + public void test_performSync_withPluginsToUpgrade_callsRemoveAndInstallAction() throws Exception { + final InstallPluginAction installAction = mock(InstallPluginAction.class); + final RemovePluginAction removeAction = mock(RemovePluginAction.class); + final InOrder inOrder = Mockito.inOrder(removeAction, installAction); + + final List pluginDescriptors = Arrays.asList(new PluginDescriptor("plugin1"), new PluginDescriptor("plugin2")); + + action.performSync(installAction, removeAction, new PluginChanges(emptyList(), emptyList(), pluginDescriptors)); + + inOrder.verify(removeAction).setPurge(false); + inOrder.verify(removeAction).execute(pluginDescriptors); + inOrder.verify(installAction).execute(pluginDescriptors); + } + + /** + * Check that if there are plugins to remove, install and upgrade, then we do everything. + */ + public void test_performSync_withPluginsToUpgrade_callsUpgradeAction() throws Exception { + final InstallPluginAction installAction = mock(InstallPluginAction.class); + final RemovePluginAction removeAction = mock(RemovePluginAction.class); + final InOrder inOrder = Mockito.inOrder(removeAction, installAction); + + final List pluginsToRemove = Collections.singletonList(new PluginDescriptor("plugin1")); + final List pluginsToInstall = Collections.singletonList(new PluginDescriptor("plugin2")); + final List pluginsToUpgrade = Collections.singletonList(new PluginDescriptor("plugin3")); + + action.performSync(installAction, removeAction, new PluginChanges(pluginsToRemove, pluginsToInstall, pluginsToUpgrade)); + + inOrder.verify(removeAction).setPurge(true); + inOrder.verify(removeAction).execute(pluginsToRemove); + + inOrder.verify(installAction).execute(pluginsToInstall); + + inOrder.verify(removeAction).setPurge(false); + inOrder.verify(removeAction).execute(pluginsToUpgrade); + inOrder.verify(installAction).execute(pluginsToUpgrade); + } + + private void createPlugin(String name) throws IOException { + createPlugin(name, Version.CURRENT); + } + + private void createPlugin(String name, Version version) throws IOException { + PluginTestUtil.writePluginProperties( + env.pluginsFile().resolve(name), + "description", + "dummy", + "name", + name, + "version", + "1.0", + "elasticsearch.version", + version.toString(), + "java.version", + System.getProperty("java.specification.version"), + "classname", + "SomeClass" + ); + } +} diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java index a8a7e42c6fa71..ddc9e459b877a 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java @@ -27,7 +27,7 @@ * The available methods are similar to those of {@link Console}, with the ability * to read either normal text or a password, and the ability to print a line * of text. Printing is also gated by the {@link Verbosity} of the terminal, - * which allows {@link #println(Verbosity,String)} calls which act like a logger, + * which allows {@link #println(Verbosity,CharSequence)} calls which act like a logger, * only actually printing if the verbosity level of the terminal is above * the verbosity of the message. */ @@ -113,7 +113,7 @@ public final void print(Verbosity verbosity, String msg) { } /** Prints message to the terminal at {@code verbosity} level, without a newline. */ - private void print(Verbosity verbosity, String msg, boolean isError) { + protected void print(Verbosity verbosity, String msg, boolean isError) { if (isPrintable(verbosity)) { PrintWriter writer = isError ? getErrorWriter() : getWriter(); writer.print(msg); @@ -206,6 +206,16 @@ public void flush() { this.getErrorWriter().flush(); } + /** + * Indicates whether this terminal is for a headless system i.e. is not interactive. If an instances answers + * {@code false}, interactive operations can be attempted, but it is not guaranteed that they will succeed. + * + * @return if this terminal is headless. + */ + public boolean isHeadless() { + return false; + } + private static class ConsoleTerminal extends Terminal { private static final Console CONSOLE = System.console(); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 8aa1ebbefbf25..c303638ee9dcc 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.packaging.util.Shell; import org.elasticsearch.packaging.util.Shell.Result; import org.elasticsearch.packaging.util.docker.DockerRun; +import org.elasticsearch.packaging.util.docker.MockServer; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -34,10 +35,12 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.StringJoiner; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.nio.file.attribute.PosixFilePermissions.fromString; +import static java.util.Arrays.asList; import static org.elasticsearch.packaging.util.Distribution.Packaging; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; @@ -47,6 +50,7 @@ import static org.elasticsearch.packaging.util.FileMatcher.p755; import static org.elasticsearch.packaging.util.FileMatcher.p775; import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.FileUtils.deleteIfExists; import static org.elasticsearch.packaging.util.FileUtils.rm; import static org.elasticsearch.packaging.util.docker.Docker.chownWithPrivilegeEscalation; import static org.elasticsearch.packaging.util.docker.Docker.copyFromContainer; @@ -70,9 +74,11 @@ import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; @@ -96,6 +102,9 @@ public class DockerTests extends PackagingTestCase { private Path tempDir; + private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin"; + private static final String EXAMPLE_PLUGIN_PATH = System.getProperty(EXAMPLE_PLUGIN_SYSPROP); + @BeforeClass public static void filterDistros() { assumeTrue("only Docker", distribution().isDocker()); @@ -147,47 +156,199 @@ public void test020PluginsListWithNoPlugins() { /** * Check that Cloud images bundle a selection of plugins. */ - public void test021PluginsListWithPlugins() { + public void test021PluginsListWithDefaultCloudPlugins() { assumeTrue( "Only applies to Cloud images", distribution.packaging == Packaging.DOCKER_CLOUD || distribution().packaging == Packaging.DOCKER_CLOUD_ESS ); final Installation.Executables bin = installation.executables(); - final List plugins = Arrays.asList(sh.run(bin.pluginTool + " list").stdout.split("\n")); + final List plugins = asList(sh.run(bin.pluginTool + " list").stdout.split("\n")); assertThat( "Expected standard plugins to be listed", plugins, - equalTo(Arrays.asList("repository-azure", "repository-gcs", "repository-s3")) + equalTo(asList("repository-azure", "repository-gcs", "repository-s3")) ); } /** - * Checks that ESS images can install plugins from the local archive. + * Check that a plugin can be installed without special permissions. */ - public void test022InstallPluginsFromLocalArchive() { - assumeTrue("Only applies to ESS images", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); + public void test022InstallPlugin() { + runContainer(distribution(), builder().volume(Paths.get(EXAMPLE_PLUGIN_PATH), "/analysis-icu.zip")); final String plugin = "analysis-icu"; + assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin))); + + final Installation.Executables bin = installation.executables(); + sh.run(bin.pluginTool + " install file:///analysis-icu.zip"); + + final boolean isCloudImage = distribution().packaging == Packaging.DOCKER_CLOUD + || distribution().packaging == Packaging.DOCKER_CLOUD_ESS; + + final List expectedPlugins = isCloudImage + ? asList("analysis-icu", "repository-azure", "repository-gcs", "repository-s3") + : asList("analysis-icu"); + + assertThat("Expected installed plugins to be listed", listPlugins(), equalTo(expectedPlugins)); + } + + /** + * Checks that ESS images can install plugins from the local archive. + */ + public void test023InstallPluginsFromLocalArchive() { + assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); + final String plugin = "analysis-icu"; final Installation.Executables bin = installation.executables(); - List plugins = Arrays.asList(sh.run(bin.pluginTool + " list").stdout.split("\n")); - assertThat("Expected " + plugin + " to not be installed", plugins, not(hasItems(plugin))); + assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin))); // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail sh.getEnv() .put("ES_JAVA_OPTS", "-Dhttp.proxyHost=example.org -Dhttp.proxyPort=9999 -Dhttps.proxyHost=example.org -Dhttps.proxyPort=9999"); - sh.run("/opt/plugins/plugin-wrapper.sh install --batch analysis-icu"); + sh.run(bin.pluginTool + " install --batch analysis-icu"); + + assertThat("Expected " + plugin + " to be installed", listPlugins(), hasItems(plugin)); + } + + /** + * Checks that plugins can be installed by deploying a plugins config file. + */ + public void test024InstallPluginUsingConfigFile() throws Exception { + final boolean isCloudImage = distribution().packaging == Packaging.DOCKER_CLOUD + || distribution().packaging == Packaging.DOCKER_CLOUD_ESS; + + final StringJoiner pluginsDescriptor = new StringJoiner("\n", "", "\n"); + pluginsDescriptor.add("plugins:"); + pluginsDescriptor.add(" - id: analysis-icu"); + pluginsDescriptor.add(" location: file:///analysis-icu.zip"); + if (isCloudImage) { + // The repository plugins have to be present, because (1) they are preinstalled, and (2) they + // are owned by `root` and can't be removed. + Stream.of("repository-s3", "repository-azure", "repository-gcs").forEach(plugin -> pluginsDescriptor.add(" - id: " + plugin)); + } + + final String filename = "elasticsearch-plugins.yml"; + append(tempDir.resolve(filename), pluginsDescriptor.toString()); + + // Restart the container. This will sync the plugins automatically. Also + // stuff the proxy settings with garbage, so any attempt to go out to the internet would fail. The + // command should instead use the bundled plugin archive. + runContainer( + distribution(), + builder().volume(tempDir.resolve(filename), installation.config.resolve(filename)) + .volume(Paths.get(EXAMPLE_PLUGIN_PATH), "/analysis-icu.zip") + .envVar( + "ES_JAVA_OPTS", + "-Dhttp.proxyHost=example.org -Dhttp.proxyPort=9999 -Dhttps.proxyHost=example.org -Dhttps.proxyPort=9999" + ) + ); - plugins = Arrays.asList(sh.run(bin.pluginTool + " list").stdout.split("\n")); + // Since ES is doing the installing, give it a chance to complete + waitForElasticsearch(installation); + + assertThat("List of installed plugins is incorrect", listPlugins(), hasItems("analysis-icu")); + } + + /** + * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file. + */ + public void test025InstallPluginFromArchiveUsingConfigFile() throws Exception { + assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); + + // The repository plugins have to be present, because (1) they are preinstalled, and (2) they + // are owned by `root` and can't be removed. + final String[] plugins = { "repository-s3", "repository-azure", "repository-gcs", "analysis-icu", "analysis-phonetic" }; + + final StringJoiner pluginsDescriptor = new StringJoiner("\n", "", "\n"); + pluginsDescriptor.add("plugins:"); + for (String plugin : plugins) { + pluginsDescriptor.add(" - id: " + plugin); + } + + final String filename = "elasticsearch-plugins.yml"; + append(tempDir.resolve(filename), pluginsDescriptor.toString()); + + // Restart the container. This will sync the plugins automatically. Also + // stuff the proxy settings with garbage, so any attempt to go out to the internet would fail. The + // command should instead use the bundled plugin archive. + runContainer( + distribution(), + builder().volume(tempDir.resolve(filename), installation.config.resolve(filename)) + .envVar( + "ES_JAVA_OPTS", + "-Dhttp.proxyHost=example.org -Dhttp.proxyPort=9999 -Dhttps.proxyHost=example.org -Dhttps.proxyPort=9999" + ) + ); + + // Since ES is doing the installing, give it a chance to complete + waitForElasticsearch(installation); + + assertThat("List of installed plugins is incorrect", listPlugins(), containsInAnyOrder(plugins)); + } - assertThat("Expected " + plugin + " to be installed", plugins, hasItems(plugin)); + /** + * Check that when using Elasticsearch's plugins sync capability, it will use a proxy when configured to do so. + * This could either be in the plugins config file, or via the standard Java system properties. + */ + public void test026SyncPluginsUsingProxy() { + MockServer.withMockServer(mockServer -> { + for (boolean useConfigFile : asList(true, false)) { + mockServer.clearExpectations(); + + final StringJoiner config = new StringJoiner("\n", "", "\n"); + config.add("plugins:"); + // The repository plugins have to be present for Cloud images, because (1) they are preinstalled, and (2) they + // are owned by `root` and can't be removed. + if (distribution().packaging == Packaging.DOCKER_CLOUD || distribution().packaging == Packaging.DOCKER_CLOUD_ESS) { + for (String plugin : asList("repository-s3", "repository-azure", "repository-gcs", "analysis-icu")) { + config.add(" - id: " + plugin); + } + } + // This is the new plugin to install. We don't use an official plugin because then Elasticsearch + // will attempt an SSL connection and that just makes everything more complicated. + config.add(" - id: my-plugin"); + config.add(" location: http://example.com/my-plugin.zip"); + + if (useConfigFile) { + config.add("proxy: mockserver:" + mockServer.getPort()); + } + + final String filename = "elasticsearch-plugins.yml"; + final Path pluginsConfigPath = tempDir.resolve(filename); + deleteIfExists(pluginsConfigPath); + append(pluginsConfigPath, config.toString()); + + final DockerRun builder = builder().volume(pluginsConfigPath, installation.config.resolve(filename)) + .extraArgs("--link " + mockServer.getContainerId() + ":mockserver"); + + if (useConfigFile == false) { + builder.envVar("ES_JAVA_OPTS", "-Dhttp.proxyHost=mockserver -Dhttp.proxyPort=" + mockServer.getPort()); + } + + // Restart the container. This will sync plugins automatically, which will fail because + // ES will be unable to install `my-plugin` + final Result result = runContainerExpectingFailure(distribution(), builder); + + final List> interactions = mockServer.getInteractions(); + + assertThat(result.stderr, containsString("FileNotFoundException: http://example.com/my-plugin.zip")); + + // Now check that Elasticsearch did use the proxy server + assertThat(interactions, hasSize(1)); + final Map interaction = interactions.get(0); + assertThat(interaction, hasEntry("httpRequest.headers.Host[0]", "example.com")); + assertThat(interaction, hasEntry("httpRequest.headers.User-Agent[0]", "elasticsearch-plugin-installer")); + assertThat(interaction, hasEntry("httpRequest.method", "GET")); + assertThat(interaction, hasEntry("httpRequest.path", "/my-plugin.zip")); + } + }); } /** - * Check that the JDK's cacerts file is a symlink to the copy provided by the operating system. + * Check that the JDK's `cacerts` file is a symlink to the copy provided by the operating system. */ public void test040JavaUsesTheOsProvidedKeystore() { final String path = sh.run("realpath jdk/lib/security/cacerts").stdout; @@ -1025,4 +1186,9 @@ public void test400CloudImageBundlesBeats() { assertThat(Paths.get("/opt/" + beat + "/modules.d"), file(Directory, "root", "root", p755)); }); } + + private List listPlugins() { + final Installation.Executables bin = installation.executables(); + return Arrays.stream(sh.run(bin.pluginTool + " list").stdout.split("\n")).collect(Collectors.toList()); + } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java index d744103f28534..9094406b22207 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java @@ -9,12 +9,14 @@ package org.elasticsearch.packaging.test; import org.apache.http.client.fluent.Request; -import org.elasticsearch.packaging.test.PackagingTestCase.AwaitsFix; +import org.elasticsearch.packaging.util.FileUtils; import org.elasticsearch.packaging.util.Installation; import org.elasticsearch.packaging.util.Platforms; import org.elasticsearch.packaging.util.Shell; import org.junit.Before; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -22,17 +24,19 @@ import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.matchesPattern; import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; -@AwaitsFix(bugUrl = "Needs to be re-enabled") +@PackagingTestCase.AwaitsFix(bugUrl = "Needs to be re-enabled") public class PluginCliTests extends PackagingTestCase { private static final String EXAMPLE_PLUGIN_NAME = "custom-settings"; private static final Path EXAMPLE_PLUGIN_ZIP; static { // re-read before each test so the plugin path can be manipulated within tests - EXAMPLE_PLUGIN_ZIP = Paths.get(System.getProperty("tests.example-plugin", "/dummy/path")); + EXAMPLE_PLUGIN_ZIP = Paths.get(System.getProperty("tests.example-plugin")); } @Before @@ -47,7 +51,7 @@ public interface PluginAction { private Shell.Result assertWithPlugin(Installation.Executable pluginTool, Path pluginZip, String pluginName, PluginAction action) throws Exception { - Shell.Result installResult = pluginTool.run("install --batch \"" + pluginZip.toUri().toString() + "\""); + Shell.Result installResult = pluginTool.run("install --batch \"" + pluginZip.toUri() + "\""); action.run(installResult); return pluginTool.run("remove " + pluginName); } @@ -69,6 +73,7 @@ public void test20SymlinkPluginsDir() throws Exception { Path linkedPlugins = createTempDir("symlinked-plugins"); Platforms.onLinux(() -> sh.run("chown elasticsearch:elasticsearch " + linkedPlugins.toString())); Files.createSymbolicLink(pluginsDir, linkedPlugins); + // Packaged installation don't get autoconfigured yet assertWithExamplePlugin(installResult -> { assertWhileRunning(() -> { final String pluginsResponse = makeRequest(Request.Get("http://localhost:9200/_cat/plugins?h=component")).trim(); @@ -116,4 +121,47 @@ public void test25Umask() throws Exception { sh.setUmask("0077"); assertWithExamplePlugin(installResult -> {}); } + + /** + * Check that the `install` subcommand cannot be used if a plugins config file exists. + */ + public void test30InstallFailsIfConfigFilePresent() throws IOException { + Files.write(installation.config.resolve("elasticsearch-plugins.yml"), "".getBytes(StandardCharsets.UTF_8)); + + Shell.Result result = installation.executables().pluginTool.run("install analysis-icu", null, true); + assertThat(result.isSuccess(), is(false)); + assertThat(result.stderr, matchesPattern("^Plugins config \\[[^+]] exists.*")); + } + + /** + * Check that the `remove` subcommand cannot be used if a plugins config file exists. + */ + public void test31RemoveFailsIfConfigFilePresent() throws IOException { + Files.write(installation.config.resolve("elasticsearch-plugins.yml"), "".getBytes(StandardCharsets.UTF_8)); + + Shell.Result result = installation.executables().pluginTool.run("install analysis-icu", null, true); + assertThat(result.isSuccess(), is(false)); + assertThat(result.stderr, matchesPattern("^Plugins config \\[[^+]] exists.*")); + } + + /** + * Check that when a plugins config file exists, Elasticsearch refuses to start up, since using + * a config file is only supported in Docker. + */ + public void test32FailsToStartWhenPluginsConfigExists() throws Exception { + try { + Files.write( + installation.config("elasticsearch-plugins.yml"), + "content doesn't matter for this test".getBytes(StandardCharsets.UTF_8) + ); + Shell.Result result = runElasticsearchStartCommand(null, false, true); + assertThat(result.isSuccess(), equalTo(false)); + assertThat( + result.stderr, + containsString("Can only use [elasticsearch-plugins.yml] config file with distribution type [docker]") + ); + } finally { + FileUtils.rm(installation.config("elasticsearch-plugins.yml")); + } + } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index e8e6f1061ac0f..2e632855d78b2 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -158,6 +158,10 @@ public Shell.Result run(String args) { } public Shell.Result run(String args, String input) { + return run(args, input, false); + } + + public Shell.Result run(String args, String input, boolean ignoreExitCode) { String command = path.toString(); if (Platforms.WINDOWS) { command = "& '" + command + "'"; @@ -171,6 +175,9 @@ public Shell.Result run(String args, String input) { if (input != null) { command = "echo \"" + input + "\" | " + command; } + if (ignoreExitCode) { + return sh.runIgnoreExitCode(command + " " + args); + } return sh.run(command + " " + args); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index a937fd794b7cc..71296d780719c 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -193,7 +193,7 @@ private static void waitForElasticsearchToExit() { do { try { // Give the container a chance to exit out - Thread.sleep(1000); + Thread.sleep(2000); if (sh.run("docker ps --quiet --no-trunc").stdout.contains(containerId) == false) { isElasticsearchRunning = false; @@ -435,8 +435,6 @@ public static void verifyContainerInstallation(Installation es) { } private static void verifyCloudContainerInstallation(Installation es) { - assertThat(Paths.get("/opt/plugins/plugin-wrapper.sh"), file("root", "root", p555)); - final String pluginArchive = "/opt/plugins/archive"; final List plugins = listContents(pluginArchive); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index ab6aa6a70d514..3df891e9e3d4c 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -11,6 +11,7 @@ import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Platforms; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -54,6 +55,10 @@ public DockerRun envVar(String key, String value) { } public DockerRun volume(Path from, String to) { + requireNonNull(from); + if (Files.exists(from) == false) { + throw new RuntimeException("Path [" + from + "] does not exist"); + } this.volumes.put(requireNonNull(from), Paths.get(requireNonNull(to))); return this; } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java new file mode 100644 index 0000000000000..5cdf0dcfec978 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/MockServer.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.packaging.util.docker; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.packaging.test.PackagingTestCase; +import org.elasticsearch.packaging.util.Shell; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Providers an interface to Mockserver, where a proxy + * server is needed for testing in Docker tests. + *

+ * To use the server, link the container under test with the mockserver using the --link + * CLI option, using the {@link #getContainerId()} option. By aliasing the ID, you will know what + * hostname to use to connect to the proxy. For example: + * + * 

"--link " + mockserver.getContainerId() + ":mockserver"
+ * + *

All requests will result in a 404, but those requests are recorded and can be retried with + * {@link #getInteractions()}. These can can be reset with {@link #clearExpectations()}. + */ +public class MockServer { + protected final Logger logger = LogManager.getLogger(getClass()); + + private static final int CONTAINER_PORT = 1080; // default for image + + private final Shell shell; + private CloseableHttpClient client; + private String containerId; + + /** + * Create a new mockserver, and execute the supplied {@code runnable}. The mockserver will + * be cleaned up afterwards. + * @param runnable the code to run e.g. the test case + */ + public static void withMockServer(CheckedConsumer runnable) { + final MockServer mockServer = new MockServer(); + try { + mockServer.start(); + runnable.accept(mockServer); + mockServer.close(); + } catch (Throwable e) { + mockServer.close(); + } + } + + private MockServer() { + this.shell = new Shell(); + this.client = HttpClients.createDefault(); + } + + private void start() throws Exception { + final String command = "docker run -t --detach --rm -p " + CONTAINER_PORT + ":" + CONTAINER_PORT + " mockserver/mockserver:latest"; + this.containerId = this.shell.run(command).stdout.trim(); + + // It's a Java app, so give it a chance to wake up. I'd add a healthcheck to the above command, + // but the image doesn't have any CLI utils at all. + PackagingTestCase.assertBusy(() -> { + try { + this.reset(); + } catch (Exception e) { + // Only assertions are retried. + throw new AssertionError(e); + } + }, 20, TimeUnit.SECONDS); + } + + public void clearExpectations() throws Exception { + doRequest("http://localhost:" + CONTAINER_PORT + "/mockserver/clear?type=EXPECTATIONS", "{ \"path\": \"/*\" }"); + } + + public void reset() throws Exception { + doRequest("http://localhost:" + CONTAINER_PORT + "/mockserver/reset", null); + } + + /** + * Returns all interactions with the mockserver since startup, the last call to {@link #reset()} or the + * last call to {@link #clearExpectations()}. The JSON returned by the mockserver is flattened, so that + * the period-seperated keys in each map represent the structure of the JSON. + * + * @return a list of interactions + * @throws Exception if anything goes wrong + */ + public List> getInteractions() throws Exception { + final String url = "http://localhost:" + CONTAINER_PORT + "/mockserver/retrieve?type=REQUEST_RESPONSES"; + + final String result = doRequest(url, null); + + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonNode = objectMapper.readTree(result); + + assertThat("Response from mockserver is not a JSON array", jsonNode.isArray(), is(true)); + + final List> interactions = new ArrayList<>(); + + for (JsonNode node : jsonNode) { + final Map interaction = new HashMap<>(); + addKeys("", node, interaction); + interactions.add(interaction); + } + + return interactions; + } + + private void close() { + if (this.containerId != null) { + this.shell.run("docker rm -f " + this.containerId); + this.containerId = null; + } + + if (this.client != null) { + try { + this.client.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + this.client = null; + } + } + + public String getContainerId() { + return containerId; + } + + public int getPort() { + return CONTAINER_PORT; + } + + /** + * Recursively flattens a JsonNode into a map, to make it easier to pick out entries and make assertions. + * Keys are concatenated with periods. + * + * @param currentPath used recursively to construct the key + * @param jsonNode the current node to flatten + * @param map entries are added into this map + */ + private void addKeys(String currentPath, JsonNode jsonNode, Map map) { + if (jsonNode.isObject()) { + ObjectNode objectNode = (ObjectNode) jsonNode; + Iterator> iter = objectNode.fields(); + String pathPrefix = currentPath.isEmpty() ? "" : currentPath + "."; + + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + addKeys(pathPrefix + entry.getKey(), entry.getValue(), map); + } + } else if (jsonNode.isArray()) { + ArrayNode arrayNode = (ArrayNode) jsonNode; + for (int i = 0; i < arrayNode.size(); i++) { + addKeys(currentPath + "[" + i + "]", arrayNode.get(i), map); + } + } else if (jsonNode.isValueNode()) { + ValueNode valueNode = (ValueNode) jsonNode; + map.put(currentPath, valueNode.asText()); + } + } + + private String doRequest(String urlString, String body) throws Exception { + final HttpPost httpPost = new HttpPost(urlString); + + if (body != null) { + httpPost.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON)); + } + + try (CloseableHttpResponse response = client.execute(httpPost)) { + return EntityUtils.toString(response.getEntity()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index fd06fa71dbb54..e7853357287dd 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -332,6 +332,7 @@ private static Version fromStringSlow(String version) { public final byte revision; public final byte build; public final org.apache.lucene.util.Version luceneVersion; + private final int previousMajorId; Version(int id, org.apache.lucene.util.Version luceneVersion) { this.id = id; @@ -340,6 +341,7 @@ private static Version fromStringSlow(String version) { this.revision = (byte) ((id / 100) % 100); this.build = (byte) (id % 100); this.luceneVersion = Objects.requireNonNull(luceneVersion); + this.previousMajorId = major > 0 ? (major - 1) * 1000000 + 99 : major; } public boolean after(Version version) { @@ -461,6 +463,14 @@ public boolean isCompatible(Version version) { return compatible; } + /** + * Returns a first major version previous to the version stored in this object. + * I.e 8.1.0 will return 7.0.0 + */ + public Version previousMajor() { + return Version.fromId(previousMajorId); + } + @SuppressForbidden(reason = "System.out.*") public static void main(String[] args) { final String versionOutput = String.format( diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index e69bfb864c864..4226cdbed16c0 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -16,8 +16,10 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.lucene.util.Constants; import org.apache.lucene.util.StringHelper; +import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.plugins.PluginsManager; import org.elasticsearch.cli.KeyStoreAwareCommand; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; @@ -415,6 +417,20 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui // setDefaultUncaughtExceptionHandler Thread.setDefaultUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler()); + if (PluginsManager.configExists(environment)) { + if (Build.CURRENT.type() == Build.Type.DOCKER) { + try { + PluginsManager.syncPlugins(environment); + } catch (Exception e) { + throw new BootstrapException(e); + } + } else { + throw new BootstrapException( + new ElasticsearchException("Can only use [elasticsearch-plugins.yml] config file with distribution type [docker]") + ); + } + } + INSTANCE.setup(true, environment); try { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/plugins/LoggerTerminal.java b/server/src/main/java/org/elasticsearch/bootstrap/plugins/LoggerTerminal.java new file mode 100644 index 0000000000000..4a216a57db4cf --- /dev/null +++ b/server/src/main/java/org/elasticsearch/bootstrap/plugins/LoggerTerminal.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.bootstrap.plugins; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.spi.AbstractLogger; +import org.apache.logging.log4j.spi.ExtendedLoggerWrapper; +import org.elasticsearch.cli.Terminal; + +import java.io.OutputStream; +import java.io.PrintWriter; + +public final class LoggerTerminal extends Terminal { + private final ExtendedLoggerWrapper logger; + + private static final String FQCN = LoggerTerminal.class.getName(); + + private LoggerTerminal(final Logger logger) { + super(System.lineSeparator()); + this.logger = new ExtendedLoggerWrapper((AbstractLogger) logger, logger.getName(), logger.getMessageFactory()); + } + + public static LoggerTerminal getLogger(String logger) { + return new LoggerTerminal(LogManager.getLogger(logger)); + } + + @Override + public boolean isHeadless() { + return true; + } + + @Override + public String readText(String prompt) { + throw new UnsupportedOperationException(); + } + + @Override + public char[] readSecret(String prompt) { + throw new UnsupportedOperationException(); + } + + @Override + public char[] readSecret(String prompt, int maxLength) { + throw new UnsupportedOperationException(); + } + + @Override + public PrintWriter getWriter() { + throw new UnsupportedOperationException(); + } + + @Override + public OutputStream getOutputStream() { + throw new UnsupportedOperationException(); + } + + @Override + public PrintWriter getErrorWriter() { + throw new UnsupportedOperationException(); + } + + @Override + protected void print(Verbosity verbosity, String msg, boolean isError) { + Level level; + switch (verbosity) { + case SILENT: + level = isError ? Level.ERROR : Level.WARN; + break; + + case VERBOSE: + level = Level.DEBUG; + break; + + case NORMAL: + default: + level = isError ? Level.WARN : Level.INFO; + break; + } + this.logger.logIfEnabled(FQCN, level, null, msg.trim(), (Throwable) null); + } + + @Override + public void flush() { + throw new UnsupportedOperationException(); + } +} diff --git a/server/src/main/java/org/elasticsearch/bootstrap/plugins/PluginsManager.java b/server/src/main/java/org/elasticsearch/bootstrap/plugins/PluginsManager.java new file mode 100644 index 0000000000000..82a9b1a00235d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/bootstrap/plugins/PluginsManager.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.bootstrap.plugins; + +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.PluginsSynchronizer; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * This class is responsible for adding, updating or removing plugins so that the list of installed plugins + * matches those in the {@code elasticsearch-plugins.yml} config file. It does this by loading a class + * dynamically from the {@code plugin-cli} jar and executing it. + */ +public class PluginsManager { + + public static final String SYNC_PLUGINS_ACTION = "org.elasticsearch.plugins.cli.SyncPluginsAction"; + + public static boolean configExists(Environment env) { + return Files.exists(env.configFile().resolve("elasticsearch-plugins.yml")); + } + + /** + * Synchronizes the currently-installed plugins. + * @param env the environment to use + * @throws Exception if anything goes wrong + */ + public static void syncPlugins(Environment env) throws Exception { + ClassLoader classLoader = buildClassLoader(env); + + @SuppressWarnings("unchecked") + final Class synchronizerClass = (Class) classLoader.loadClass(SYNC_PLUGINS_ACTION); + + final PluginsSynchronizer provider = synchronizerClass.getConstructor(Terminal.class, Environment.class) + .newInstance(LoggerTerminal.getLogger(SYNC_PLUGINS_ACTION), env); + + provider.execute(); + } + + private static ClassLoader buildClassLoader(Environment env) { + final Path pluginLibDir = env.libFile().resolve("tools").resolve("plugin-cli"); + + try { + final URL[] urls = Files.list(pluginLibDir) + .filter(each -> each.getFileName().toString().endsWith(".jar")) + .map(PluginsManager::pathToURL) + .toArray(URL[]::new); + + return URLClassLoader.newInstance(urls, PluginsManager.class.getClassLoader()); + } catch (IOException e) { + throw new RuntimeException("Failed to list jars in [" + pluginLibDir + "]: " + e.getMessage(), e); + } + } + + private static URL pathToURL(Path path) { + try { + return path.toUri().toURL(); + } catch (MalformedURLException e) { + // Shouldn't happen, but have to handle the exception + throw new RuntimeException("Failed to convert path [" + path + "] to URL", e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 19e4966da6777..1de332219abd2 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -363,10 +363,13 @@ public static List findPluginDirs(final Path rootPath) throws IOException if (Files.exists(rootPath)) { try (DirectoryStream stream = Files.newDirectoryStream(rootPath)) { for (Path plugin : stream) { - if (FileSystemUtils.isDesktopServicesStore(plugin) || plugin.getFileName().toString().startsWith(".removing-")) { + final String filename = plugin.getFileName().toString(); + if (FileSystemUtils.isDesktopServicesStore(plugin) + || filename.startsWith(".removing-") + || filename.equals(".elasticsearch-plugins.yml.cache")) { continue; } - if (seen.add(plugin.getFileName().toString()) == false) { + if (seen.add(filename) == false) { throw new IllegalStateException("duplicate plugin: " + plugin); } plugins.add(plugin); diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsSynchronizer.java b/server/src/main/java/org/elasticsearch/plugins/PluginsSynchronizer.java new file mode 100644 index 0000000000000..5a13b19993da9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsSynchronizer.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins; + +/** + * This is a marker interface for classes that are capable of synchronizing the currently-installed ES plugins + * with those that ought to be installed according to a configuration file. + */ +public interface PluginsSynchronizer { + void execute() throws Exception; +}