Introduce declarative plugin management

build-tools-internal/src/main/groovy/elasticsearch.local-distribution.gradle

@@ -13,6 +13,7 @@
  * build/distributions/local
  * */
 import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.VersionProperties
 
 // gradle has an open issue of failing applying plugins in
 // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004)
@@ -29,6 +30,6 @@ tasks.register('localDistro', Sync) {
   from(elasticsearch_distributions.local)
   into("build/distribution/local")
   doLast {
-    logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}.")
+    logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}/elasticsearch-${VersionProperties.elasticsearch}")
   }
 }

build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml

@@ -34,7 +34,7 @@
         <message key="javadoc.missing" value="Types should explain their purpose" />
     </module>
 
-    <!-- Public methods must have JavaDoc -->
+    <!-- Check the Javadoc for a method e.g that it has the correct parameters, return type etc -->
     <module name="JavadocMethod">
         <property name="severity" value="warning"/>
         <property name="accessModifiers" value="public"/>

distribution/build.gradle

@@ -295,7 +295,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
      *****************************************************************************/
     libFiles =
       copySpec {
-        // delay by using closures, since they have not yet been configured, so no jar task exists yet
+        // Delay by using closures, since they have not yet been configured, so no jar task exists yet.
         from(configurations.libs)
         into('tools/geoip-cli') {
           from(configurations.libsGeoIpCli)

distribution/docker/build.gradle

@@ -265,12 +265,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         }
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
-
-        into('bin') {
-          from(project.projectDir.toPath().resolve('src/docker/cloud')) {
-            expand([ version: VersionProperties.elasticsearch ])
-          }
-        }
       }
 
       onlyIf { Architecture.current() == architecture }

distribution/docker/src/docker/Dockerfile

@@ -262,12 +262,17 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     find config -type f -exec chmod 0664 {} +
 
 <% if (docker_base == "cloud") { %>
-# Preinstall common plugins
+# Preinstall common plugins. Note that these are installed as root, meaning the `elasticsearch` user cannot delete them.
 COPY repository-s3-${version}.zip repository-gcs-${version}.zip repository-azure-${version}.zip /tmp/
-RUN bin/elasticsearch-plugin install --batch \\
+RUN bin/elasticsearch-plugin install --batch --verbose \\
     file:/tmp/repository-s3-${version}.zip \\
     file:/tmp/repository-gcs-${version}.zip \\
     file:/tmp/repository-azure-${version}.zip
+# Generate a replacement example plugins config that reflects what is actually installed
+RUN echo "plugins:" > config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-azure" >> config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-gcs"   >> config/elasticsearch-plugins.example.yml && \\
+    echo "  - id: repository-s3"    >> config/elasticsearch-plugins.example.yml
 
 <% /*  I tried to use `ADD` here, but I couldn't force it to do what I wanted */ %>
 COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
@@ -277,8 +282,6 @@ RUN mkdir -p /opt/filebeat /opt/metricbeat && \\
 
 # Add plugins infrastructure
 RUN mkdir -p /opt/plugins/archive
-COPY bin/plugin-wrapper.sh /opt/plugins
-# These are the correct permissions for both the directories and the script
 RUN chmod -R 0555 /opt/plugins
 <% } %>
 

distribution/docker/src/docker/Dockerfile.cloud-ess

@@ -10,3 +10,4 @@ RUN chmod 0444 /opt/plugins/archive/*
 FROM ${base_image}
 
 COPY --from=builder /opt/plugins /opt/plugins
+ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

distribution/packages/build.gradle

@@ -175,6 +175,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // ========= config files =========
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
+    configurationFile '/etc/elasticsearch/elasticsearch-plugins.example.yml'
     configurationFile '/etc/elasticsearch/jvm.options'
     configurationFile '/etc/elasticsearch/log4j2.properties'
     configurationFile '/etc/elasticsearch/role_mapping.yml'

distribution/src/config/elasticsearch-plugins.example.yml

@@ -0,0 +1,26 @@
+# Rename this file to `elasticsearch-plugins.yml` to use it.
+#
+# All plugins must be listed here. If you add a plugin to this list and run
+# `elasticsearch-plugin sync`, that plugin will be installed. If you remove
+# a plugin and re-run the command, that plugin will be removed.
+
+plugins:
+  # Each plugin must have an ID. Plugins with only an ID are official plugins and will be downloaded from Elastic.
+  - id: example-id
+
+  # Plugins can be specified by URL (it doesn't have to be HTTP, you could use e.g. `file:`)
+  - id: example-with-url
+    location: https://some.domain/path/example4.zip
+
+  # Or by maven coordinates:
+  - id: example-with-maven-url
+    location: org.elasticsearch.plugins:example-plugin:1.2.3
+
+  # A proxy can also be configured per-plugin, if necessary
+  - id: example-with-proxy
+    location: https://some.domain/path/example.zip
+    proxy: https://some.domain:1234
+
+# Configures a proxy for all network access. Remove this if you don't need
+# to use a proxy.
+proxy: https://some.domain:1234

distribution/tools/plugin-cli/build.gradle

@@ -13,10 +13,6 @@ archivesBaseName = 'elasticsearch-plugin-cli'
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:elasticsearch-cli")
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
   api "org.bouncycastle:bcpg-fips:1.0.4"
   api "org.bouncycastle:bc-fips:1.0.2"
   testImplementation project(":test:framework")

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java

@@ -15,6 +15,9 @@
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginInfo;
+import org.elasticsearch.plugins.cli.action.InstallPluginAction;
+import org.elasticsearch.plugins.cli.action.PluginDescriptor;
+import org.elasticsearch.plugins.cli.action.SyncPluginsAction;
 
 import java.util.Arrays;
 import java.util.List;
@@ -75,13 +78,17 @@ protected void printAdditionalHelp(Terminal terminal) {
 
     @Override
     protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        SyncPluginsAction.ensureNoConfigFile(env);
+
         List<PluginDescriptor> plugins = arguments.values(options)
             .stream()
-            .map(id -> new PluginDescriptor(id, id))
+            // We only have one piece of data, which could be an ID or could be a location, so we use it for both
+            .map(idOrLocation -> new PluginDescriptor(idOrLocation, idOrLocation))
             .collect(Collectors.toList());
         final boolean isBatch = options.has(batchOption);
 
-        InstallPluginAction action = new InstallPluginAction(terminal, env, isBatch);
-        action.execute(plugins);
+        try (InstallPluginAction action = new InstallPluginAction(terminal, env, isBatch)) {
+            action.execute(plugins);
+        }
     }
 }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ProgressInputStream.java

@@ -20,13 +20,13 @@
  *
  * Only used by the InstallPluginCommand, thus package private here
  */
-abstract class ProgressInputStream extends FilterInputStream {
+public abstract class ProgressInputStream extends FilterInputStream {
 
     private final int expectedTotalSize;
     private int currentPercent;
     private int count = 0;
 
-    ProgressInputStream(InputStream is, int expectedTotalSize) {
+    public ProgressInputStream(InputStream is, int expectedTotalSize) {
         super(is);
         this.expectedTotalSize = expectedTotalSize;
         this.currentPercent = 0;

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java

@@ -14,6 +14,9 @@
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.cli.action.PluginDescriptor;
+import org.elasticsearch.plugins.cli.action.RemovePluginAction;
+import org.elasticsearch.plugins.cli.action.SyncPluginsAction;
 
 import java.util.Arrays;
 import java.util.List;
@@ -34,6 +37,8 @@ class RemovePluginCommand extends EnvironmentAwareCommand {
 
     @Override
     protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws Exception {
+        SyncPluginsAction.ensureNoConfigFile(env);
+
         final List<PluginDescriptor> plugins = arguments.values(options).stream().map(PluginDescriptor::new).collect(Collectors.toList());
 
         final RemovePluginAction action = new RemovePluginAction(terminal, env, options.has(purgeOption));