From f9248352656f80a355f3da374ea87d705647bd41 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 28 Jun 2018 08:24:03 -0700 Subject: [PATCH 01/18] Core: Require all actions have a Task (#31627) The TaskManager and TaskAwareRequest could return null when registering a task according to their javadocs, but no implementations ever actually did that. This commit removes that wording from the javadocs and ensures null is no longer allowed. --- .../action/support/TransportAction.java | 28 +++++----- .../elasticsearch/tasks/TaskAwareRequest.java | 3 -- .../org/elasticsearch/tasks/TaskManager.java | 7 +-- .../transport/RequestHandlerRegistry.java | 18 +++---- .../node/tasks/TransportTasksActionTests.java | 53 +++---------------- .../test/tasks/MockTaskManager.java | 20 ++++--- .../TransportMonitoringBulkActionTests.java | 5 +- 7 files changed, 39 insertions(+), 95 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java index 9db5bfd84b5e3..e861a9f8d3f6c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -59,23 +59,19 @@ public final Task execute(Request request, ActionListener listener) { * this method. */ Task task = taskManager.register("transport", actionName, request); - if (task == null) { - execute(null, request, listener); - } else { - execute(task, request, new ActionListener() { - @Override - public void onResponse(Response response) { - taskManager.unregister(task); - listener.onResponse(response); - } + execute(task, request, new ActionListener() { + @Override + public void onResponse(Response response) { + taskManager.unregister(task); + listener.onResponse(response); + } - @Override - public void onFailure(Exception e) { - taskManager.unregister(task); - listener.onFailure(e); - } - }); - } + @Override + public void onFailure(Exception e) { + taskManager.unregister(task); + listener.onFailure(e); + } + }); return task; } diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java b/server/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java index 86ba59ebcc804..0b9dfba3e834f 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java @@ -45,9 +45,6 @@ default void setParentTask(String parentTaskNode, long parentTaskId) { /** * Returns the task object that should be used to keep track of the processing of the request. - * - * A request can override this method and return null to avoid being tracked by the task - * manager. */ default Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new Task(id, type, action, getDescription(), parentTaskId, headers); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java index 80427b197239d..73af26cfc708b 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java @@ -45,6 +45,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; @@ -91,8 +92,6 @@ public void setTaskResultsService(TaskResultsService taskResultsService) { /** * Registers a task without parent task - *

- * Returns the task manager tracked task or null if the task doesn't support the task manager */ public Task register(String type, String action, TaskAwareRequest request) { Map headers = new HashMap<>(); @@ -110,9 +109,7 @@ public Task register(String type, String action, TaskAwareRequest request) { } } Task task = request.createTask(taskIdGenerator.incrementAndGet(), type, action, request.getParentTask(), headers); - if (task == null) { - return null; - } + Objects.requireNonNull(task); assert task.getParentTaskId().equals(request.getParentTask()) : "Request [ " + request + "] didn't preserve it parentTaskId"; if (logger.isTraceEnabled()) { logger.trace("register {} [{}] [{}] [{}]", task.getId(), type, action, task.getDescription()); diff --git a/server/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java b/server/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java index 4e09daf9ccf0a..7887dd2c7caca 100644 --- a/server/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java +++ b/server/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java @@ -58,17 +58,13 @@ public Request newRequest(StreamInput in) throws IOException { public void processMessageReceived(Request request, TransportChannel channel) throws Exception { final Task task = taskManager.register(channel.getChannelType(), action, request); - if (task == null) { - handler.messageReceived(request, channel, null); - } else { - boolean success = false; - try { - handler.messageReceived(request, new TaskTransportChannel(taskManager, task, channel), task); - success = true; - } finally { - if (success == false) { - taskManager.unregister(task); - } + boolean success = false; + try { + handler.messageReceived(request, new TaskTransportChannel(taskManager, task, channel), task); + success = true; + } finally { + if (success == false) { + taskManager.unregister(task); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 9175bc69bf642..edc79db79422d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -79,7 +79,6 @@ public class TransportTasksActionTests extends TaskManagerTestCase { public static class NodeRequest extends BaseNodeRequest { protected String requestName; - private boolean enableTaskManager; public NodeRequest() { super(); @@ -88,82 +87,63 @@ public NodeRequest() { public NodeRequest(NodesRequest request, String nodeId) { super(nodeId); requestName = request.requestName; - enableTaskManager = request.enableTaskManager; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); requestName = in.readString(); - enableTaskManager = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(requestName); - out.writeBoolean(enableTaskManager); } @Override public String getDescription() { - return "CancellableNodeRequest[" + requestName + ", " + enableTaskManager + "]"; + return "CancellableNodeRequest[" + requestName + "]"; } @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - if (enableTaskManager) { - return super.createTask(id, type, action, parentTaskId, headers); - } else { - return null; - } + return super.createTask(id, type, action, parentTaskId, headers); } } public static class NodesRequest extends BaseNodesRequest { private String requestName; - private boolean enableTaskManager; NodesRequest() { super(); } public NodesRequest(String requestName, String... nodesIds) { - this(requestName, true, nodesIds); - } - - public NodesRequest(String requestName, boolean enableTaskManager, String... nodesIds) { super(nodesIds); this.requestName = requestName; - this.enableTaskManager = enableTaskManager; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); requestName = in.readString(); - enableTaskManager = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(requestName); - out.writeBoolean(enableTaskManager); } @Override public String getDescription() { - return "CancellableNodesRequest[" + requestName + ", " + enableTaskManager + "]"; + return "CancellableNodesRequest[" + requestName + "]"; } @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - if (enableTaskManager) { - return super.createTask(id, type, action, parentTaskId, headers); - } else { - return null; - } + return super.createTask(id, type, action, parentTaskId, headers); } } @@ -400,7 +380,7 @@ public void onFailure(Exception e) { assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); - assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription()); + assertEquals("CancellableNodeRequest[Test Request]", entry.getValue().get(0).getDescription()); } // Make sure that the main task on coordinating node is the task that was returned to us by execute() @@ -455,27 +435,6 @@ public void testFindChildTasks() throws Exception { assertEquals(0, responses.failureCount()); } - public void testTaskManagementOptOut() throws Exception { - setupTestNodes(Settings.EMPTY); - connectNodes(testNodes); - CountDownLatch checkLatch = new CountDownLatch(1); - // Starting actions that disable task manager - ActionFuture future = startBlockingTestNodesAction(checkLatch, new NodesRequest("Test Request", false)); - - TestNode testNode = testNodes[randomIntBetween(0, testNodes.length - 1)]; - - // Get the parent task - ListTasksRequest listTasksRequest = new ListTasksRequest(); - listTasksRequest.setActions("testAction*"); - ListTasksResponse response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); - assertEquals(0, response.getTasks().size()); - - // Release all tasks and wait for response - checkLatch.countDown(); - NodesResponse responses = future.get(); - assertEquals(0, responses.failureCount()); - } - public void testTasksDescriptions() throws Exception { long minimalStartTime = System.currentTimeMillis(); setupTestNodes(Settings.EMPTY); @@ -502,7 +461,7 @@ public void testTasksDescriptions() throws Exception { assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); - assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription()); + assertEquals("CancellableNodeRequest[Test Request]", entry.getValue().get(0).getDescription()); assertThat(entry.getValue().get(0).getStartTime(), greaterThanOrEqualTo(minimalStartTime)); assertThat(entry.getValue().get(0).getRunningTimeNanos(), greaterThanOrEqualTo(minimalDurationNanos)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java index 41cdaefe03575..0133a8be0c0bd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java @@ -50,17 +50,15 @@ public MockTaskManager(Settings settings, ThreadPool threadPool, Set tas @Override public Task register(String type, String action, TaskAwareRequest request) { Task task = super.register(type, action, request); - if (task != null) { - for (MockTaskManagerListener listener : listeners) { - try { - listener.onTaskRegistered(task); - } catch (Exception e) { - logger.warn( - (Supplier) () -> new ParameterizedMessage( - "failed to notify task manager listener about registering the task with id {}", - task.getId()), - e); - } + for (MockTaskManagerListener listener : listeners) { + try { + listener.onTaskRegistered(task); + } catch (Exception e) { + logger.warn( + (Supplier) () -> new ParameterizedMessage( + "failed to notify task manager listener about registering the task with id {}", + task.getId()), + e); } } return task; diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index 61e99d2dc8585..55393b1860be3 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.monitoring.action; -import java.util.concurrent.ExecutorService; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilter; @@ -28,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskAwareRequest; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; @@ -51,6 +51,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.ExecutorService; import static org.elasticsearch.Version.CURRENT; import static org.hamcrest.Matchers.containsString; @@ -98,7 +99,7 @@ public void setUpMocks() { filters = mock(ActionFilters.class); when(transportService.getTaskManager()).thenReturn(taskManager); - when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(null); + when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executor); From 101d675f904a13caa84bae891aebdfc6e45a8445 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Thu, 28 Jun 2018 08:27:04 -0700 Subject: [PATCH 02/18] [DOCS] Replace CONFIG_DIR with ES_PATH_CONF (#31635) --- docs/reference/settings/security-settings.asciidoc | 6 +++--- .../en/security/auditing/output-logfile.asciidoc | 2 +- .../authentication/configuring-file-realm.asciidoc | 6 +++--- .../authentication/configuring-ldap-realm.asciidoc | 2 +- .../security/authorization/managing-roles.asciidoc | 2 +- x-pack/docs/en/security/reference/files.asciidoc | 12 ++++++------ .../security/securing-communications/tls-ad.asciidoc | 2 +- .../securing-communications/tls-ldap.asciidoc | 2 +- x-pack/docs/en/security/troubleshooting.asciidoc | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index a98bca676152d..ba2fc27ed4263 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -328,7 +328,7 @@ role mappings are not considered. Defaults to `false`. `files.role_mapping`:: The {xpack-ref}/security-files.html[location] for the {xpack-ref}/mapping-roles.html#mapping-roles[ YAML role mapping configuration file]. Defaults to -`CONFIG_DIR/role_mapping.yml`. +`ES_PATH_CONF/role_mapping.yml`. `follow_referrals`:: Specifies whether {security} should follow referrals returned @@ -494,7 +494,7 @@ considered. Defaults to `false`. `files.role_mapping`:: The {xpack-ref}/security-files.html[location] for the YAML -role mapping configuration file. Defaults to `CONFIG_DIR/role_mapping.yml`. +role mapping configuration file. Defaults to `ES_PATH_CONF/role_mapping.yml`. `user_search.base_dn`:: The context to search for a user. Defaults to the root @@ -719,7 +719,7 @@ for SSL. This setting cannot be used with `certificate_authorities`. `files.role_mapping`:: Specifies the {xpack-ref}/security-files.html[location] of the {xpack-ref}/mapping-roles.html[YAML role mapping configuration file]. -Defaults to `CONFIG_DIR/role_mapping.yml`. +Defaults to `ES_PATH_CONF/role_mapping.yml`. `cache.ttl`:: Specifies the time-to-live for cached user entries. A user and a hash of its diff --git a/x-pack/docs/en/security/auditing/output-logfile.asciidoc b/x-pack/docs/en/security/auditing/output-logfile.asciidoc index 849046bdb9db0..ee33f618f9665 100644 --- a/x-pack/docs/en/security/auditing/output-logfile.asciidoc +++ b/x-pack/docs/en/security/auditing/output-logfile.asciidoc @@ -47,7 +47,7 @@ audited in plain text when including the request body in audit events. [[logging-file]] You can also configure how the logfile is written in the `log4j2.properties` -file located in `CONFIG_DIR`. By default, audit information is appended to the +file located in `ES_PATH_CONF`. By default, audit information is appended to the `_access.log` file located in the standard Elasticsearch `logs` directory (typically located at `$ES_HOME/logs`). The file rolls over on a daily basis. diff --git a/x-pack/docs/en/security/authentication/configuring-file-realm.asciidoc b/x-pack/docs/en/security/authentication/configuring-file-realm.asciidoc index 8555902e503d3..683da76bb7b99 100644 --- a/x-pack/docs/en/security/authentication/configuring-file-realm.asciidoc +++ b/x-pack/docs/en/security/authentication/configuring-file-realm.asciidoc @@ -5,7 +5,7 @@ You can manage and authenticate users with the built-in `file` internal realm. All the data about the users for the `file` realm is stored in two files on each node in the cluster: `users` and `users_roles`. Both files are located in -`CONFIG_DIR/` and are read on startup. +`ES_PATH_CONF` and are read on startup. [IMPORTANT] ============================== @@ -50,7 +50,7 @@ xpack: . Restart {es}. -. Add user information to the `CONFIG_DIR/users` file on each node in the +. Add user information to the `ES_PATH_CONF/users` file on each node in the cluster. + -- @@ -76,7 +76,7 @@ IMPORTANT: As the administrator of the cluster, it is your responsibility to -- -. Add role information to the `CONFIG_DIR/users_roles` file on each node +. Add role information to the `ES_PATH_CONF/users_roles` file on each node in the cluster. + -- diff --git a/x-pack/docs/en/security/authentication/configuring-ldap-realm.asciidoc b/x-pack/docs/en/security/authentication/configuring-ldap-realm.asciidoc index 6ea9b243aad4d..e32c9eb5300b3 100644 --- a/x-pack/docs/en/security/authentication/configuring-ldap-realm.asciidoc +++ b/x-pack/docs/en/security/authentication/configuring-ldap-realm.asciidoc @@ -56,7 +56,7 @@ xpack: group_search: base_dn: "dc=example,dc=com" files: - role_mapping: "CONFIG_DIR/role_mapping.yml" + role_mapping: "ES_PATH_CONF/role_mapping.yml" unmapped_groups_as_roles: false ------------------------------------------------------------ diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index 6ee5d9d39bbf2..b8a01aa4519c6 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -137,7 +137,7 @@ see {ref}/security-api-roles.html[Role Management APIs]. === File-based role management Apart from the _Role Management APIs_, roles can also be defined in local -`roles.yml` file located in `CONFIG_DIR`. This is a YAML file where each +`roles.yml` file located in `ES_PATH_CONF`. This is a YAML file where each role definition is keyed by its name. [IMPORTANT] diff --git a/x-pack/docs/en/security/reference/files.asciidoc b/x-pack/docs/en/security/reference/files.asciidoc index dcf673d9a9f26..ac4cc0aa201ee 100644 --- a/x-pack/docs/en/security/reference/files.asciidoc +++ b/x-pack/docs/en/security/reference/files.asciidoc @@ -1,23 +1,23 @@ [[security-files]] === Security Files -The {security} uses the following files: +{security} uses the following files: -* `CONFIG_DIR/roles.yml` defines the roles in use on the cluster +* `ES_PATH_CONF/roles.yml` defines the roles in use on the cluster (read more <>). -* `CONFIG_DIR/elasticsearch-users` defines the users and their hashed passwords for +* `ES_PATH_CONF/elasticsearch-users` defines the users and their hashed passwords for the <>. -* `CONFIG_DIR/elasticsearch-users_roles` defines the user roles assignment for the +* `ES_PATH_CONF/elasticsearch-users_roles` defines the user roles assignment for the the <>. -* `CONFIG_DIR/role_mapping.yml` defines the role assignments for a +* `ES_PATH_CONF/role_mapping.yml` defines the role assignments for a Distinguished Name (DN) to a role. This allows for LDAP and Active Directory groups and users and PKI users to be mapped to roles (read more <>). -* `CONFIG_DIR/log4j2.properties` contains audit information (read more +* `ES_PATH_CONF/log4j2.properties` contains audit information (read more <>). [[security-files-location]] diff --git a/x-pack/docs/en/security/securing-communications/tls-ad.asciidoc b/x-pack/docs/en/security/securing-communications/tls-ad.asciidoc index d189501f1e2a5..2421925f08ebe 100644 --- a/x-pack/docs/en/security/securing-communications/tls-ad.asciidoc +++ b/x-pack/docs/en/security/securing-communications/tls-ad.asciidoc @@ -37,7 +37,7 @@ xpack: domain_name: ad.example.com url: ldaps://ad.example.com:636 ssl: - certificate_authorities: [ "CONFIG_DIR/cacert.pem" ] + certificate_authorities: [ "ES_PATH_CONF/cacert.pem" ] -------------------------------------------------- The CA cert must be a PEM encoded certificate. diff --git a/x-pack/docs/en/security/securing-communications/tls-ldap.asciidoc b/x-pack/docs/en/security/securing-communications/tls-ldap.asciidoc index b7f0b7d300590..1ffc667cd33c1 100644 --- a/x-pack/docs/en/security/securing-communications/tls-ldap.asciidoc +++ b/x-pack/docs/en/security/securing-communications/tls-ldap.asciidoc @@ -29,7 +29,7 @@ xpack: order: 0 url: "ldaps://ldap.example.com:636" ssl: - certificate_authorities: [ "CONFIG_DIR/cacert.pem" ] + certificate_authorities: [ "ES_PATH_CONF/cacert.pem" ] -------------------------------------------------- The CA certificate must be a PEM encoded. diff --git a/x-pack/docs/en/security/troubleshooting.asciidoc b/x-pack/docs/en/security/troubleshooting.asciidoc index 622bc753d2a52..d1c88b2786f1e 100644 --- a/x-pack/docs/en/security/troubleshooting.asciidoc +++ b/x-pack/docs/en/security/troubleshooting.asciidoc @@ -101,7 +101,7 @@ The role definition might be missing or invalid. |====================== To help track down these possibilities, add the following lines to the end of -the `log4j2.properties` configuration file in the `CONFIG_DIR`: +the `log4j2.properties` configuration file in the `ES_PATH_CONF`: [source,properties] ---------------- From 1af31f441ab05cc3b422cd22a9633ef070f4cdd8 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Thu, 28 Jun 2018 16:22:35 +0000 Subject: [PATCH 03/18] [TEST] Mute failing NamingConventionsTaskIT tests Note: no awaitsFix available, so I fell back to JUnit @Ignore Tracking issue: #31665 --- .../gradle/precommit/NamingConventionsTaskIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java index 7e469e8597ddd..a8bc0ea878be0 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java @@ -4,6 +4,7 @@ import org.gradle.testkit.runner.BuildResult; import org.gradle.testkit.runner.GradleRunner; import org.gradle.testkit.runner.TaskOutcome; +import org.junit.Ignore; import java.util.Arrays; @@ -21,6 +22,7 @@ public void testPluginCanBeApplied() { assertTrue(output, output.contains("build plugin can be applied")); } + @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665") public void testNameCheckFailsAsItShould() { BuildResult result = GradleRunner.create() .withProjectDir(getProjectDir("namingConventionsSelfTest")) @@ -46,6 +48,7 @@ public void testNameCheckFailsAsItShould() { } } + @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665") public void testNameCheckFailsAsItShouldWithMain() { BuildResult result = GradleRunner.create() .withProjectDir(getProjectDir("namingConventionsSelfTest")) From db6b33978e613f27f1d71178649a3e528972bff7 Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Thu, 28 Jun 2018 19:39:39 +0300 Subject: [PATCH 04/18] Configurable password hashing algorithm/cost (#31234) Make password hashing algorithm/cost configurable for the stored passwords of users for the realms that this applies (native, reserved). Replaces predefined choice of bcrypt with cost factor 10. This also introduces PBKDF2 with configurable cost (number of iterations) as an algorithm option for password hashing both for storing passwords and for the user cache. Password hash validation algorithm selection takes into consideration the stored hash prefix and only a specific number of algorithnm and cost factor options for brypt and pbkdf2 are whitelisted and can be selected in the relevant setting. --- .../common/settings/Setting.java | 12 + .../xpack/core/XPackSettings.java | 20 + .../user/ChangePasswordRequestBuilder.java | 13 +- .../action/user/PutUserRequestBuilder.java | 10 +- .../CachingUsernamePasswordRealmSettings.java | 3 +- .../core/security/authc/support/Hasher.java | 424 ++++++++++++++---- .../core/security/client/SecurityClient.java | 20 +- .../xpack/core/XPackSettingsTests.java | 2 - ...asswordHashingAlgorithmBootstrapCheck.java | 41 ++ .../xpack/security/Security.java | 7 +- .../user/TransportChangePasswordAction.java | 9 + .../authc/esnative/NativeUsersStore.java | 20 +- .../authc/esnative/ReservedRealm.java | 27 +- .../authc/esnative/UserAndPassword.java | 8 + .../authc/file/FileUserPasswdStore.java | 6 +- .../security/authc/file/tool/UsersTool.java | 11 +- .../support/CachingUsernamePasswordRealm.java | 10 +- .../action/user/RestChangePasswordAction.java | 5 +- .../rest/action/user/RestPutUserAction.java | 6 +- .../AbstractPrivilegeTestCase.java | 3 - .../integration/ClearRealmsCacheTests.java | 6 +- .../integration/ClusterPrivilegeTests.java | 15 +- .../DateMathExpressionIntegTests.java | 6 +- .../DocumentAndFieldLevelSecurityTests.java | 14 +- .../DocumentLevelSecurityRandomTests.java | 6 +- .../DocumentLevelSecurityTests.java | 9 +- .../FieldLevelSecurityRandomTests.java | 12 +- .../integration/FieldLevelSecurityTests.java | 19 +- .../integration/IndexPrivilegeTests.java | 36 +- ...onsWithAliasesWildcardsAndRegexsTests.java | 5 +- .../integration/KibanaUserRoleIntegTests.java | 5 +- .../MultipleIndicesPermissionsTests.java | 22 +- .../PermissionPrecedenceTests.java | 9 +- .../integration/SecurityClearScrollTests.java | 7 +- .../elasticsearch/license/LicensingTests.java | 9 +- .../test/SecurityIntegTestCase.java | 6 +- .../test/SecuritySettingsSource.java | 6 +- ...rdHashingAlgorithmBootstrapCheckTests.java | 44 ++ .../user/PutUserRequestBuilderTests.java | 13 +- .../TransportChangePasswordActionTests.java | 64 ++- .../user/TransportPutUserActionTests.java | 3 +- .../security/authc/RealmSettingsTests.java | 9 +- .../esnative/ESNativeMigrateToolTests.java | 2 +- .../authc/esnative/NativeRealmIntegTests.java | 81 ++-- .../authc/esnative/NativeUsersStoreTests.java | 5 +- .../esnative/ReservedRealmIntegTests.java | 2 +- .../authc/esnative/ReservedRealmTests.java | 79 ++-- .../security/authc/file/FileRealmTests.java | 7 +- .../authc/file/FileUserPasswdStoreTests.java | 41 +- .../CachingUsernamePasswordRealmTests.java | 22 +- .../security/authc/support/HasherTests.java | 93 +++- .../xpack/security/authz/AnalyzeTests.java | 8 +- .../security/authz/IndexAliasesTests.java | 15 +- .../security/authz/SecurityScrollTests.java | 4 +- .../SecurityIndexManagerIntegTests.java | 3 +- .../xpack/security/authc/file/users | 7 +- .../example/role/CustomRolesProviderIT.java | 3 +- .../xpack/security/MigrateToolIT.java | 3 +- .../authc/file/tool/UsersToolTests.java | 16 +- 59 files changed, 982 insertions(+), 391 deletions(-) create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheck.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheckTests.java diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 7f3906ff5a251..94edb5a297afa 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1017,6 +1017,18 @@ public static Setting simpleString(String key, Validator validat return new Setting<>(new SimpleKey(key), null, s -> "", Function.identity(), validator, properties); } + /** + * Creates a new Setting instance with a String value + * + * @param key the settings key for this setting. + * @param defaultValue the default String value. + * @param properties properties for this setting like scope, filtering... + * @return the Setting Object + */ + public static Setting simpleString(String key, String defaultValue, Property... properties) { + return new Setting<>(key, s -> defaultValue, Function.identity(), properties); + } + public static int parseInt(String s, int minValue, String key) { return parseInt(s, minValue, Integer.MAX_VALUE, key); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index ec747247a58c5..b0d0c4f2c2eea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.xpack.core.security.SecurityField; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.ssl.SSLClientAuth; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.core.ssl.VerificationMode; @@ -19,6 +20,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; +import java.util.function.Function; import static org.elasticsearch.xpack.core.security.SecurityField.USER_SETTING; @@ -26,6 +29,11 @@ * A container for xpack setting constants. */ public class XPackSettings { + + private XPackSettings() { + throw new IllegalStateException("Utility class should not be instantiated"); + } + /** Setting for enabling or disabling security. Defaults to true. */ public static final Setting SECURITY_ENABLED = Setting.boolSetting("xpack.security.enabled", true, Setting.Property.NodeScope); @@ -105,6 +113,17 @@ public class XPackSettings { DEFAULT_CIPHERS = ciphers; } + /* + * Do not allow insecure hashing algorithms to be used for password hashing + */ + public static final Setting PASSWORD_HASHING_ALGORITHM = new Setting<>( + "xpack.security.authc.password_hashing.algorithm", "bcrypt", Function.identity(), (v, s) -> { + if (Hasher.getAvailableAlgoStoredHash().contains(v.toLowerCase(Locale.ROOT)) == false) { + throw new IllegalArgumentException("Invalid algorithm: " + v + ". Only pbkdf2 or bcrypt family algorithms can be used for " + + "password hashing."); + } + }, Setting.Property.NodeScope); + public static final List DEFAULT_SUPPORTED_PROTOCOLS = Arrays.asList("TLSv1.2", "TLSv1.1", "TLSv1"); public static final SSLClientAuth CLIENT_AUTH_DEFAULT = SSLClientAuth.REQUIRED; public static final SSLClientAuth HTTP_CLIENT_AUTH_DEFAULT = SSLClientAuth.NONE; @@ -143,6 +162,7 @@ public static List> getAllSettings() { settings.add(SQL_ENABLED); settings.add(USER_SETTING); settings.add(ROLLUP_ENABLED); + settings.add(PASSWORD_HASHING_ALGORITHM); return Collections.unmodifiableList(settings); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java index 48f71d2cffcae..d7538c2a556f0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java @@ -41,22 +41,22 @@ public ChangePasswordRequestBuilder username(String username) { return this; } - public static char[] validateAndHashPassword(SecureString password) { + public static char[] validateAndHashPassword(SecureString password, Hasher hasher) { Validation.Error error = Validation.Users.validatePassword(password.getChars()); if (error != null) { ValidationException validationException = new ValidationException(); validationException.addValidationError(error.toString()); throw validationException; } - return Hasher.BCRYPT.hash(password); + return hasher.hash(password); } /** * Sets the password. Note: the char[] passed to this method will be cleared. */ - public ChangePasswordRequestBuilder password(char[] password) { + public ChangePasswordRequestBuilder password(char[] password, Hasher hasher) { try (SecureString secureString = new SecureString(password)) { - char[] hash = validateAndHashPassword(secureString); + char[] hash = validateAndHashPassword(secureString, hasher); request.passwordHash(hash); } return this; @@ -65,7 +65,8 @@ public ChangePasswordRequestBuilder password(char[] password) { /** * Populate the change password request from the source in the provided content type */ - public ChangePasswordRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException { + public ChangePasswordRequestBuilder source(BytesReference source, XContentType xContentType, Hasher hasher) throws + IOException { // EMPTY is ok here because we never call namedObject try (InputStream stream = source.streamInput(); XContentParser parser = xContentType.xContent() @@ -80,7 +81,7 @@ public ChangePasswordRequestBuilder source(BytesReference source, XContentType x if (token == XContentParser.Token.VALUE_STRING) { String password = parser.text(); final char[] passwordChars = password.toCharArray(); - password(passwordChars); + password(passwordChars, hasher); assert CharBuffer.wrap(passwordChars).chars().noneMatch((i) -> (char) i != (char) 0) : "expected password to " + "clear the char[] but it did not!"; } else { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java index 60964a7490dac..eea804d81fe9f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesReference; @@ -33,8 +32,6 @@ public class PutUserRequestBuilder extends ActionRequestBuilder implements WriteRequestBuilder { - private final Hasher hasher = Hasher.BCRYPT; - public PutUserRequestBuilder(ElasticsearchClient client) { this(client, PutUserAction.INSTANCE); } @@ -53,7 +50,7 @@ public PutUserRequestBuilder roles(String... roles) { return this; } - public PutUserRequestBuilder password(@Nullable char[] password) { + public PutUserRequestBuilder password(char[] password, Hasher hasher) { if (password != null) { Validation.Error error = Validation.Users.validatePassword(password); if (error != null) { @@ -96,7 +93,8 @@ public PutUserRequestBuilder enabled(boolean enabled) { /** * Populate the put user request using the given source and username */ - public PutUserRequestBuilder source(String username, BytesReference source, XContentType xContentType) throws IOException { + public PutUserRequestBuilder source(String username, BytesReference source, XContentType xContentType, Hasher hasher) throws + IOException { Objects.requireNonNull(xContentType); username(username); // EMPTY is ok here because we never call namedObject @@ -113,7 +111,7 @@ public PutUserRequestBuilder source(String username, BytesReference source, XCon if (token == XContentParser.Token.VALUE_STRING) { String password = parser.text(); char[] passwordChars = password.toCharArray(); - password(passwordChars); + password(passwordChars, hasher); Arrays.fill(passwordChars, (char) 0); } else { throw new ElasticsearchParseException( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingUsernamePasswordRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingUsernamePasswordRealmSettings.java index a1d031a5b00c0..6d060b0febbd4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingUsernamePasswordRealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingUsernamePasswordRealmSettings.java @@ -13,7 +13,8 @@ import java.util.Set; public final class CachingUsernamePasswordRealmSettings { - public static final Setting CACHE_HASH_ALGO_SETTING = Setting.simpleString("cache.hash_algo", Setting.Property.NodeScope); + public static final Setting CACHE_HASH_ALGO_SETTING = Setting.simpleString("cache.hash_algo", "ssha256", + Setting.Property.NodeScope); private static final TimeValue DEFAULT_TTL = TimeValue.timeValueMinutes(20); public static final Setting CACHE_TTL_SETTING = Setting.timeSetting("cache.ttl", DEFAULT_TTL, Setting.Property.NodeScope); private static final int DEFAULT_MAX_USERS = 100_000; //100k users diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java index 0d4a1d23e7910..f5275de5fc887 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java @@ -5,15 +5,22 @@ */ package org.elasticsearch.xpack.core.security.authc.support; -import org.elasticsearch.common.Randomness; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.SecureString; -import java.nio.charset.StandardCharsets; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import java.nio.CharBuffer; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.spec.InvalidKeySpecException; +import java.util.Arrays; import java.util.Base64; +import java.util.List; import java.util.Locale; -import java.util.Random; +import java.util.stream.Collectors; public enum Hasher { @@ -26,11 +33,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -43,11 +46,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -60,11 +59,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -77,11 +72,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -94,11 +85,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -111,11 +98,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); } }, @@ -128,12 +111,164 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { - String hashStr = new String(hash); - if (!hashStr.startsWith(BCRYPT_PREFIX)) { - return false; - } - return BCrypt.checkpw(text, hashStr); + return verifyBcryptHash(text, hash); + } + }, + + BCRYPT10() { + @Override + public char[] hash(SecureString text) { + String salt = BCrypt.gensalt(10); + return BCrypt.hashpw(text, salt).toCharArray(); + } + + @Override + public boolean verify(SecureString text, char[] hash) { + return verifyBcryptHash(text, hash); + } + }, + + BCRYPT11() { + @Override + public char[] hash(SecureString text) { + String salt = BCrypt.gensalt(11); + return BCrypt.hashpw(text, salt).toCharArray(); + } + + @Override + public boolean verify(SecureString text, char[] hash) { + return verifyBcryptHash(text, hash); + } + }, + + BCRYPT12() { + @Override + public char[] hash(SecureString text) { + String salt = BCrypt.gensalt(12); + return BCrypt.hashpw(text, salt).toCharArray(); + } + + @Override + public boolean verify(SecureString text, char[] hash) { + return verifyBcryptHash(text, hash); + } + }, + + BCRYPT13() { + @Override + public char[] hash(SecureString text) { + String salt = BCrypt.gensalt(13); + return BCrypt.hashpw(text, salt).toCharArray(); + } + + @Override + public boolean verify(SecureString text, char[] hash) { + return verifyBcryptHash(text, hash); + } + }, + + BCRYPT14() { + @Override + public char[] hash(SecureString text) { + String salt = BCrypt.gensalt(14); + return BCrypt.hashpw(text, salt).toCharArray(); + } + + @Override + public boolean verify(SecureString text, char[] hash) { + return verifyBcryptHash(text, hash); + } + }, + + PBKDF2() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, PBKDF2_DEFAULT_COST); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); } + + }, + + PBKDF2_1000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 1000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + + }, + + PBKDF2_10000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 10000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + + }, + + PBKDF2_50000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 50000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + + }, + + PBKDF2_100000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 100000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + + }, + + PBKDF2_500000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 500000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + + }, + + PBKDF2_1000000() { + @Override + public char[] hash(SecureString data) { + return getPbkdf2Hash(data, 1000000); + } + + @Override + public boolean verify(SecureString data, char[] hash) { + return verifyPbkdf2Hash(data, hash); + } + }, SHA1() { @@ -149,7 +284,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { String hashStr = new String(hash); - if (!hashStr.startsWith(SHA1_PREFIX)) { + if (hashStr.startsWith(SHA1_PREFIX) == false) { return false; } byte[] textBytes = CharArrays.toUtf8Bytes(text.getChars()); @@ -173,7 +308,7 @@ public char[] hash(SecureString text) { @Override public boolean verify(SecureString text, char[] hash) { String hashStr = new String(hash); - if (!hashStr.startsWith(MD5_PREFIX)) { + if (hashStr.startsWith(MD5_PREFIX) == false) { return false; } hashStr = hashStr.substring(MD5_PREFIX.length()); @@ -189,29 +324,30 @@ public boolean verify(SecureString text, char[] hash) { public char[] hash(SecureString text) { MessageDigest md = MessageDigests.sha256(); md.update(CharArrays.toUtf8Bytes(text.getChars())); - char[] salt = SaltProvider.salt(8); - md.update(CharArrays.toUtf8Bytes(salt)); + byte[] salt = generateSalt(8); + md.update(salt); String hash = Base64.getEncoder().encodeToString(md.digest()); - char[] result = new char[SSHA256_PREFIX.length() + salt.length + hash.length()]; + char[] result = new char[SSHA256_PREFIX.length() + 12 + hash.length()]; System.arraycopy(SSHA256_PREFIX.toCharArray(), 0, result, 0, SSHA256_PREFIX.length()); - System.arraycopy(salt, 0, result, SSHA256_PREFIX.length(), salt.length); - System.arraycopy(hash.toCharArray(), 0, result, SSHA256_PREFIX.length() + salt.length, hash.length()); + System.arraycopy(Base64.getEncoder().encodeToString(salt).toCharArray(), 0, result, SSHA256_PREFIX.length(), 12); + System.arraycopy(hash.toCharArray(), 0, result, SSHA256_PREFIX.length() + 12, hash.length()); return result; } @Override public boolean verify(SecureString text, char[] hash) { String hashStr = new String(hash); - if (!hashStr.startsWith(SSHA256_PREFIX)) { + if (hashStr.startsWith(SSHA256_PREFIX) == false) { return false; } hashStr = hashStr.substring(SSHA256_PREFIX.length()); char[] saltAndHash = hashStr.toCharArray(); MessageDigest md = MessageDigests.sha256(); md.update(CharArrays.toUtf8Bytes(text.getChars())); - md.update(new String(saltAndHash, 0, 8).getBytes(StandardCharsets.UTF_8)); + // Base64 string length : (4*(n/3)) rounded up to the next multiple of 4 because of padding, 12 for 8 bytes + md.update(Base64.getDecoder().decode(new String(saltAndHash, 0, 12))); String computedHash = Base64.getEncoder().encodeToString(md.digest()); - return CharArrays.constantTimeEquals(computedHash, new String(saltAndHash, 8, saltAndHash.length - 8)); + return CharArrays.constantTimeEquals(computedHash, new String(saltAndHash, 12, saltAndHash.length - 12)); } }, @@ -231,11 +367,22 @@ public boolean verify(SecureString text, char[] hash) { private static final String SHA1_PREFIX = "{SHA}"; private static final String MD5_PREFIX = "{MD5}"; private static final String SSHA256_PREFIX = "{SSHA256}"; - - public static Hasher resolve(String name, Hasher defaultHasher) { - if (name == null) { - return defaultHasher; - } + private static final String PBKDF2_PREFIX = "{PBKDF2}"; + private static final int PBKDF2_DEFAULT_COST = 10000; + private static final int PBKDF2_KEY_LENGTH = 256; + private static final int BCRYPT_DEFAULT_COST = 10; + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + /** + * Returns a {@link Hasher} instance of the appropriate algorithm and associated cost as + * indicated by the {@code name}. Name identifiers for the default costs for + * BCRYPT and PBKDF2 return the he default BCRYPT and PBKDF2 Hasher instead of the specific + * instances for the associated cost. + * + * @param name The name of the algorithm and cost combination identifier + * @return the hasher associated with the identifier + */ + public static Hasher resolve(String name) { switch (name.toLowerCase(Locale.ROOT)) { case "bcrypt": return BCRYPT; @@ -251,6 +398,30 @@ public static Hasher resolve(String name, Hasher defaultHasher) { return BCRYPT8; case "bcrypt9": return BCRYPT9; + case "bcrypt10": + return BCRYPT; + case "bcrypt11": + return BCRYPT11; + case "bcrypt12": + return BCRYPT12; + case "bcrypt13": + return BCRYPT13; + case "bcrypt14": + return BCRYPT14; + case "pbkdf2": + return PBKDF2; + case "pbkdf2_1000": + return PBKDF2_1000; + case "pbkdf2_10000": + return PBKDF2; + case "pbkdf2_50000": + return PBKDF2_50000; + case "pbkdf2_100000": + return PBKDF2_100000; + case "pbkdf2_500000": + return PBKDF2_500000; + case "pbkdf2_1000000": + return PBKDF2_1000000; case "sha1": return SHA1; case "md5": @@ -261,38 +432,139 @@ public static Hasher resolve(String name, Hasher defaultHasher) { case "clear_text": return NOOP; default: - return defaultHasher; + throw new IllegalArgumentException("unknown hash function [" + name + "]"); } } - public static Hasher resolve(String name) { - Hasher hasher = resolve(name, null); - if (hasher == null) { - throw new IllegalArgumentException("unknown hash function [" + name + "]"); + /** + * Returns a {@link Hasher} instance that can be used to verify the {@code hash} by inspecting the + * hash prefix and determining the algorithm used for its generation. + * + * @param hash the char array from which the hashing algorithm is to be deduced + * @return the hasher that can be used for validation + */ + public static Hasher resolveFromHash(char[] hash) { + if (CharArrays.charsBeginsWith(BCRYPT_PREFIX, hash)) { + int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, BCRYPT_PREFIX.length(), hash.length - 54))); + return cost == BCRYPT_DEFAULT_COST ? Hasher.BCRYPT : resolve("bcrypt" + cost); + } else if (CharArrays.charsBeginsWith(PBKDF2_PREFIX, hash)) { + int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, PBKDF2_PREFIX.length(), hash.length - 90))); + return cost == PBKDF2_DEFAULT_COST ? Hasher.PBKDF2 : resolve("pbkdf2_" + cost); + } else if (CharArrays.charsBeginsWith(SHA1_PREFIX, hash)) { + return Hasher.SHA1; + } else if (CharArrays.charsBeginsWith(MD5_PREFIX, hash)) { + return Hasher.MD5; + } else if (CharArrays.charsBeginsWith(SSHA256_PREFIX, hash)) { + return Hasher.SSHA256; + } else { + throw new IllegalArgumentException("unknown hash format for hash [" + new String(hash) + "]"); } - return hasher; } - public abstract char[] hash(SecureString data); - - public abstract boolean verify(SecureString data, char[] hash); - - static final class SaltProvider { + /** + * Verifies that the cryptographic hash of {@code data} is the same as {@code hash}. The + * hashing algorithm and its parameters(cost factor-iterations, salt) are deduced from the + * hash itself. The {@code hash} char array is not cleared after verification. + * + * @param data the SecureString to be hashed and verified + * @param hash the char array with the hash against which the string is verified + * @return true if the hash corresponds to the data, false otherwise + */ + public static boolean verifyHash(SecureString data, char[] hash) { + try { + final Hasher hasher = resolveFromHash(hash); + return hasher.verify(data, hash); + } catch (IllegalArgumentException e) { + // The password hash format is invalid, we're unable to verify password + return false; + } + } - static final char[] ALPHABET = new char[]{ - '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', - 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', - 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' - }; + private static char[] getPbkdf2Hash(SecureString data, int cost) { + try { + // Base64 string length : (4*(n/3)) rounded up to the next multiple of 4 because of padding. + // n is 32 (PBKDF2_KEY_LENGTH in bytes) and 2 is because of the dollar sign delimiters. + CharBuffer result = CharBuffer.allocate(PBKDF2_PREFIX.length() + String.valueOf(cost).length() + 2 + 44 + 44); + result.put(PBKDF2_PREFIX); + result.put(String.valueOf(cost)); + result.put("$"); + byte[] salt = generateSalt(32); + result.put(Base64.getEncoder().encodeToString(salt)); + result.put("$"); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2withHMACSHA512"); + PBEKeySpec keySpec = new PBEKeySpec(data.getChars(), salt, cost, PBKDF2_KEY_LENGTH); + result.put(Base64.getEncoder().encodeToString(secretKeyFactory.generateSecret(keySpec).getEncoded())); + return result.array(); + } catch (InvalidKeySpecException | NoSuchAlgorithmException e) { + throw new ElasticsearchException("Can't use PBKDF2 for password hashing", e); + } + } - public static char[] salt(int length) { - Random random = Randomness.get(); - char[] salt = new char[length]; - for (int i = 0; i < length; i++) { - salt[i] = ALPHABET[(random.nextInt(ALPHABET.length))]; + private static boolean verifyPbkdf2Hash(SecureString data, char[] hash) { + // Base64 string length : (4*(n/3)) rounded up to the next multiple of 4 because of padding. + // n is 32 (PBKDF2_KEY_LENGTH in bytes), so tokenLength is 44 + final int tokenLength = 44; + char[] hashChars = null; + char[] saltChars = null; + char[] computedPwdHash = null; + try { + if (CharArrays.charsBeginsWith(PBKDF2_PREFIX, hash) == false) { + return false; + } + hashChars = Arrays.copyOfRange(hash, hash.length - tokenLength, hash.length); + saltChars = Arrays.copyOfRange(hash, hash.length - (2 * tokenLength + 1), hash.length - (tokenLength + 1)); + int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, PBKDF2_PREFIX.length(), hash.length - (2 * tokenLength + 2)))); + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2withHMACSHA512"); + PBEKeySpec keySpec = new PBEKeySpec(data.getChars(), Base64.getDecoder().decode(CharArrays.toUtf8Bytes(saltChars)), + cost, PBKDF2_KEY_LENGTH); + computedPwdHash = CharArrays.utf8BytesToChars(Base64.getEncoder() + .encode(secretKeyFactory.generateSecret(keySpec).getEncoded())); + final boolean result = CharArrays.constantTimeEquals(computedPwdHash, hashChars); + return result; + } catch (InvalidKeySpecException | NoSuchAlgorithmException e) { + throw new ElasticsearchException("Can't use PBKDF2 for password hashing", e); + } finally { + if (null != hashChars) { + Arrays.fill(hashChars, '\u0000'); + } + if (null != saltChars) { + Arrays.fill(saltChars, '\u0000'); + } + if (null != computedPwdHash) { + Arrays.fill(computedPwdHash, '\u0000'); } - return salt; } } + + private static boolean verifyBcryptHash(SecureString text, char[] hash) { + String hashStr = new String(hash); + if (hashStr.startsWith(BCRYPT_PREFIX) == false) { + return false; + } + return BCrypt.checkpw(text, hashStr); + } + + /** + * Returns a list of lower case String identifiers for the Hashing algorithm and parameter + * combinations that can be used for password hashing. The identifiers can be used to get + * an instance of the appropriate {@link Hasher} by using {@link #resolve(String) resolve()} + */ + public static List getAvailableAlgoStoredHash() { + return Arrays.stream(Hasher.values()).map(Hasher::name).map(name -> name.toLowerCase(Locale.ROOT)) + .filter(name -> (name.startsWith("pbkdf2") || name.startsWith("bcrypt"))) + .collect(Collectors.toList()); + } + + public abstract char[] hash(SecureString data); + + public abstract boolean verify(SecureString data, char[] hash); + + /** + * Generates an array of {@code length} random bytes using {@link java.security.SecureRandom} + */ + private static byte[] generateSalt(int length) { + byte[] salt = new byte[length]; + SECURE_RANDOM.nextBytes(salt); + return salt; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java index af1cfe0579e03..26c35db1fc92b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java @@ -76,6 +76,7 @@ import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequest; import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequestBuilder; import org.elasticsearch.xpack.core.security.action.user.SetEnabledResponse; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.io.IOException; import java.util.List; @@ -187,12 +188,13 @@ public void deleteUser(DeleteUserRequest request, ActionListener listener) { @@ -203,13 +205,13 @@ public void putUser(PutUserRequest request, ActionListener list * Populates the {@link ChangePasswordRequest} with the username and password. Note: the passed in char[] will be cleared by this * method. */ - public ChangePasswordRequestBuilder prepareChangePassword(String username, char[] password) { - return new ChangePasswordRequestBuilder(client).username(username).password(password); + public ChangePasswordRequestBuilder prepareChangePassword(String username, char[] password, Hasher hasher) { + return new ChangePasswordRequestBuilder(client).username(username).password(password, hasher); } - public ChangePasswordRequestBuilder prepareChangePassword(String username, BytesReference source, XContentType xContentType) - throws IOException { - return new ChangePasswordRequestBuilder(client).username(username).source(source, xContentType); + public ChangePasswordRequestBuilder prepareChangePassword(String username, BytesReference source, XContentType xContentType, + Hasher hasher) throws IOException { + return new ChangePasswordRequestBuilder(client).username(username).source(source, xContentType, hasher); } public void changePassword(ChangePasswordRequest request, ActionListener listener) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java index ed76a9a27809c..17934efe0a503 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackSettingsTests.java @@ -6,8 +6,6 @@ package org.elasticsearch.xpack.core; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.XPackSettings; - import javax.crypto.Cipher; import static org.hamcrest.Matchers.hasItem; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheck.java new file mode 100644 index 0000000000000..c60c2ea18d061 --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheck.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import org.elasticsearch.bootstrap.BootstrapCheck; +import org.elasticsearch.bootstrap.BootstrapContext; +import org.elasticsearch.xpack.core.XPackSettings; + +import javax.crypto.SecretKeyFactory; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; + +/** + * Bootstrap check to ensure that one of the allowed password hashing algorithms is + * selected and that it is available. + */ +public class PasswordHashingAlgorithmBootstrapCheck implements BootstrapCheck { + @Override + public BootstrapCheckResult check(BootstrapContext context) { + final String selectedAlgorithm = XPackSettings.PASSWORD_HASHING_ALGORITHM.get(context.settings); + if (selectedAlgorithm.toLowerCase(Locale.ROOT).startsWith("pbkdf2")) { + try { + SecretKeyFactory.getInstance("PBKDF2withHMACSHA512"); + } catch (NoSuchAlgorithmException e) { + final String errorMessage = String.format(Locale.ROOT, + "Support for PBKDF2WithHMACSHA512 must be available in order to use any of the " + + "PBKDF2 algorithms for the [%s] setting.", XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey()); + return BootstrapCheckResult.failure(errorMessage); + } + } + return BootstrapCheckResult.success(); + } + + @Override + public boolean alwaysEnforce() { + return true; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index dbb50a92f1088..e1ade053a3862 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -283,9 +283,10 @@ public Security(Settings settings, final Path configPath) { // fetched final List checks = new ArrayList<>(); checks.addAll(Arrays.asList( - new TokenSSLBootstrapCheck(), - new PkiRealmBootstrapCheck(settings, getSslService()), - new TLSLicenseBootstrapCheck())); + new TokenSSLBootstrapCheck(), + new PkiRealmBootstrapCheck(settings, getSslService()), + new TLSLicenseBootstrapCheck(), + new PasswordHashingAlgorithmBootstrapCheck())); checks.addAll(InternalRealms.getBootstrapChecks(settings, env)); this.bootstrapChecks = Collections.unmodifiableList(checks); } else { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java index 7a42cd5fdea97..5046beca1c837 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java @@ -12,9 +12,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequest; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.XPackUser; @@ -41,6 +43,13 @@ protected void doExecute(Task task, ChangePasswordRequest request, ActionListene listener.onFailure(new IllegalArgumentException("user [" + username + "] is internal")); return; } + final String requestPwdHashAlgo = Hasher.resolveFromHash(request.passwordHash()).name(); + final String configPwdHashAlgo = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)).name(); + if (requestPwdHashAlgo.equalsIgnoreCase(configPwdHashAlgo) == false) { + listener.onFailure(new IllegalArgumentException("incorrect password hashing algorithm [" + requestPwdHashAlgo + "] used while" + + " [" + configPwdHashAlgo + "] is configured.")); + return; + } nativeUsersStore.changePassword(request, new ActionListener() { @Override public void onResponse(Void v) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 72a65b8213f81..d923a0298041b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.ScrollHelper; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse; @@ -78,10 +79,9 @@ public class NativeUsersStore extends AbstractComponent { public static final String INDEX_TYPE = "doc"; static final String USER_DOC_TYPE = "user"; public static final String RESERVED_USER_TYPE = "reserved-user"; - - - private final Hasher hasher = Hasher.BCRYPT; private final Client client; + private final ReservedUserInfo disabledDefaultUserInfo; + private final ReservedUserInfo enabledDefaultUserInfo; private final SecurityIndexManager securityIndex; @@ -89,6 +89,10 @@ public NativeUsersStore(Settings settings, Client client, SecurityIndexManager s super(settings); this.client = client; this.securityIndex = securityIndex; + final char[] emptyPasswordHash = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)). + hash(new SecureString("".toCharArray())); + this.disabledDefaultUserInfo = new ReservedUserInfo(emptyPasswordHash, false, true); + this.enabledDefaultUserInfo = new ReservedUserInfo(emptyPasswordHash, true, true); } /** @@ -485,7 +489,7 @@ void verifyPassword(String username, final SecureString password, ActionListener getUserAndPassword(username, ActionListener.wrap((userAndPassword) -> { if (userAndPassword == null || userAndPassword.passwordHash() == null) { listener.onResponse(AuthenticationResult.notHandled()); - } else if (hasher.verify(password, userAndPassword.passwordHash())) { + } else if (userAndPassword.verifyPassword(password)) { listener.onResponse(AuthenticationResult.success(userAndPassword.user())); } else { listener.onResponse(AuthenticationResult.unsuccessful("Password authentication failed for " + username, null)); @@ -514,8 +518,7 @@ public void onResponse(GetResponse getResponse) { } else if (enabled == null) { listener.onFailure(new IllegalStateException("enabled must not be null!")); } else if (password.isEmpty()) { - listener.onResponse((enabled ? ReservedRealm.ENABLED_DEFAULT_USER_INFO : ReservedRealm - .DISABLED_DEFAULT_USER_INFO).deepClone()); + listener.onResponse((enabled ? enabledDefaultUserInfo : disabledDefaultUserInfo).deepClone()); } else { listener.onResponse(new ReservedUserInfo(password.toCharArray(), enabled, false)); } @@ -651,16 +654,21 @@ static final class ReservedUserInfo { public final char[] passwordHash; public final boolean enabled; public final boolean hasEmptyPassword; + private final Hasher hasher; ReservedUserInfo(char[] passwordHash, boolean enabled, boolean hasEmptyPassword) { this.passwordHash = passwordHash; this.enabled = enabled; this.hasEmptyPassword = hasEmptyPassword; + this.hasher = Hasher.resolveFromHash(this.passwordHash); } ReservedUserInfo deepClone() { return new ReservedUserInfo(Arrays.copyOf(passwordHash, passwordHash.length), enabled, hasEmptyPassword); } + boolean verifyPassword(SecureString data) { + return hasher.verify(data, this.passwordHash); + } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java index 3946a01784b16..31de03ff4330c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java @@ -49,10 +49,6 @@ public class ReservedRealm extends CachingUsernamePasswordRealm { public static final String TYPE = "reserved"; private final ReservedUserInfo bootstrapUserInfo; - static final char[] EMPTY_PASSWORD_HASH = Hasher.BCRYPT.hash(new SecureString("".toCharArray())); - static final ReservedUserInfo DISABLED_DEFAULT_USER_INFO = new ReservedUserInfo(EMPTY_PASSWORD_HASH, false, true); - static final ReservedUserInfo ENABLED_DEFAULT_USER_INFO = new ReservedUserInfo(EMPTY_PASSWORD_HASH, true, true); - public static final Setting ACCEPT_DEFAULT_PASSWORD_SETTING = Setting.boolSetting( SecurityField.setting("authc.accept_default_password"), true, Setting.Property.NodeScope, Setting.Property.Filtered, Setting.Property.Deprecated); @@ -64,6 +60,9 @@ public class ReservedRealm extends CachingUsernamePasswordRealm { private final boolean realmEnabled; private final boolean anonymousEnabled; private final SecurityIndexManager securityIndex; + private final Hasher reservedRealmHasher; + private final ReservedUserInfo disabledDefaultUserInfo; + private final ReservedUserInfo enabledDefaultUserInfo; public ReservedRealm(Environment env, Settings settings, NativeUsersStore nativeUsersStore, AnonymousUser anonymousUser, SecurityIndexManager securityIndex, ThreadPool threadPool) { @@ -73,9 +72,13 @@ public ReservedRealm(Environment env, Settings settings, NativeUsersStore native this.anonymousUser = anonymousUser; this.anonymousEnabled = AnonymousUser.isAnonymousEnabled(settings); this.securityIndex = securityIndex; - final char[] hash = BOOTSTRAP_ELASTIC_PASSWORD.get(settings).length() == 0 ? EMPTY_PASSWORD_HASH : - Hasher.BCRYPT.hash(BOOTSTRAP_ELASTIC_PASSWORD.get(settings)); - bootstrapUserInfo = new ReservedUserInfo(hash, true, hash == EMPTY_PASSWORD_HASH); + this.reservedRealmHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)); + final char[] emptyPasswordHash = reservedRealmHasher.hash(new SecureString("".toCharArray())); + disabledDefaultUserInfo = new ReservedUserInfo(emptyPasswordHash, false, true); + enabledDefaultUserInfo = new ReservedUserInfo(emptyPasswordHash, true, true); + final char[] hash = BOOTSTRAP_ELASTIC_PASSWORD.get(settings).length() == 0 ? emptyPasswordHash : + reservedRealmHasher.hash(BOOTSTRAP_ELASTIC_PASSWORD.get(settings)); + bootstrapUserInfo = new ReservedUserInfo(hash, true, hash == emptyPasswordHash); } @Override @@ -91,15 +94,15 @@ protected void doAuthenticate(UsernamePasswordToken token, ActionListener> listener) { private void getUserInfo(final String username, ActionListener listener) { if (userIsDefinedForCurrentSecurityMapping(username) == false) { logger.debug("Marking user [{}] as disabled because the security mapping is not at the required version", username); - listener.onResponse(DISABLED_DEFAULT_USER_INFO.deepClone()); + listener.onResponse(disabledDefaultUserInfo.deepClone()); } else if (securityIndex.indexExists() == false) { listener.onResponse(getDefaultUserInfo(username)); } else { @@ -212,7 +215,7 @@ private ReservedUserInfo getDefaultUserInfo(String username) { if (ElasticUser.NAME.equals(username)) { return bootstrapUserInfo.deepClone(); } else { - return ENABLED_DEFAULT_USER_INFO.deepClone(); + return enabledDefaultUserInfo.deepClone(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java index 58351ac2879b5..3f636312f0f06 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/UserAndPassword.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.security.authc.esnative; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.user.User; /** @@ -20,10 +22,12 @@ class UserAndPassword { private final User user; private final char[] passwordHash; + private final Hasher hasher; UserAndPassword(User user, char[] passwordHash) { this.user = user; this.passwordHash = passwordHash; + this.hasher = Hasher.resolveFromHash(this.passwordHash); } public User user() { @@ -34,6 +38,10 @@ public char[] passwordHash() { return this.passwordHash; } + boolean verifyPassword(SecureString data) { + return hasher.verify(data, this.passwordHash); + } + @Override public boolean equals(Object o) { return false; // Don't use this for user comparison diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java index 5773bf5a44861..15a6c2c41daed 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java @@ -46,10 +46,8 @@ public class FileUserPasswdStore { private final Logger logger; private final Path file; - private final Hasher hasher = Hasher.BCRYPT; private final Settings settings; private final CopyOnWriteArrayList listeners; - private volatile Map users; public FileUserPasswdStore(RealmConfig config, ResourceWatcherService watcherService) { @@ -84,7 +82,7 @@ public AuthenticationResult verifyPassword(String username, SecureString passwor if (hash == null) { return AuthenticationResult.notHandled(); } - if (hasher.verify(password, hash) == false) { + if (Hasher.verifyHash(password, hash) == false) { return AuthenticationResult.unsuccessful("Password authentication failed for " + username, null); } return AuthenticationResult.success(user.get()); @@ -149,7 +147,7 @@ public static Map parseFile(Path path, @Nullable Logger logger, // only trim the line because we have a format, our tool generates the formatted text and we shouldn't be lenient // and allow spaces in the format line = line.trim(); - int i = line.indexOf(":"); + int i = line.indexOf(':'); if (i <= 0 || i == line.length() - 1) { logger.error("invalid entry in users file [{}], line [{}]. skipping...", path.toAbsolutePath(), lineNr); continue; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java index 5d78a7d08f258..9d4dfba327e5d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java @@ -124,7 +124,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th if (users.containsKey(username)) { throw new UserException(ExitCodes.CODE_ERROR, "User [" + username + "] already exists"); } - Hasher hasher = Hasher.BCRYPT; + final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings())); users = new HashMap<>(users); // make modifiable users.put(username, hasher.hash(new SecureString(password))); FileUserPasswdStore.writeFile(users, passwordFile); @@ -228,7 +228,8 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th if (users.containsKey(username) == false) { throw new UserException(ExitCodes.NO_USER, "User [" + username + "] doesn't exist"); } - users.put(username, Hasher.BCRYPT.hash(new SecureString(password))); + final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings())); + users.put(username, hasher.hash(new SecureString(password))); FileUserPasswdStore.writeFile(users, file); attributesChecker.check(terminal); @@ -294,7 +295,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th Map userRolesToWrite = new HashMap<>(userRoles.size()); userRolesToWrite.putAll(userRoles); - if (roles.size() == 0) { + if (roles.isEmpty()) { userRolesToWrite.remove(username); } else { userRolesToWrite.put(username, new LinkedHashSet<>(roles).toArray(new String[]{})); @@ -367,7 +368,7 @@ static void listUsersAndRoles(Terminal terminal, Environment env, String usernam Path rolesFile = FileRolesStore.resolveFile(env).toAbsolutePath(); terminal.println(""); terminal.println(" [*] Role is not in the [" + rolesFile.toAbsolutePath() + "] file. If the role has been created " - + "using the API, please disregard this message."); + + "using the API, please disregard this message."); } } else { terminal.println(String.format(Locale.ROOT, "%-15s: -", username)); @@ -401,7 +402,7 @@ static void listUsersAndRoles(Terminal terminal, Environment env, String usernam Path rolesFile = FileRolesStore.resolveFile(env).toAbsolutePath(); terminal.println(""); terminal.println(" [*] Role is not in the [" + rolesFile.toAbsolutePath() + "] file. If the role has been created " - + "using the API, please disregard this message."); + + "using the API, please disregard this message."); } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java index e9c107abccef3..68338e9bcad93 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java @@ -32,11 +32,11 @@ public abstract class CachingUsernamePasswordRealm extends UsernamePasswordRealm private final Cache>> cache; private final ThreadPool threadPool; - final Hasher hasher; + final Hasher cacheHasher; protected CachingUsernamePasswordRealm(String type, RealmConfig config, ThreadPool threadPool) { super(type, config); - hasher = Hasher.resolve(CachingUsernamePasswordRealmSettings.CACHE_HASH_ALGO_SETTING.get(config.settings()), Hasher.SSHA256); + cacheHasher = Hasher.resolve(CachingUsernamePasswordRealmSettings.CACHE_HASH_ALGO_SETTING.get(config.settings())); this.threadPool = threadPool; TimeValue ttl = CachingUsernamePasswordRealmSettings.CACHE_TTL_SETTING.get(config.settings()); if (ttl.getNanos() > 0) { @@ -102,7 +102,7 @@ private void authenticateWithCache(UsernamePasswordToken token, ActionListener(result, userWithHash)); } else { future.onResponse(new Tuple<>(result, null)); @@ -233,16 +233,14 @@ public final void lookupUser(String username, ActionListener listener) { private static class UserWithHash { final User user; final char[] hash; - final Hasher hasher; UserWithHash(User user, SecureString password, Hasher hasher) { this.user = Objects.requireNonNull(user); this.hash = password == null ? null : hasher.hash(password); - this.hasher = hasher; } boolean verify(SecureString password) { - return hash != null && hasher.verify(password, hash); + return hash != null && Hasher.verifyHash(password, hash); } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java index b47881de2db34..1b64b3ce2baee 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java @@ -15,8 +15,10 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.rest.RestRequestFilter; import org.elasticsearch.xpack.core.security.user.User; @@ -32,6 +34,7 @@ public class RestChangePasswordAction extends SecurityBaseRestHandler implements RestRequestFilter { private final SecurityContext securityContext; + private final Hasher passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)); public RestChangePasswordAction(Settings settings, RestController controller, SecurityContext securityContext, XPackLicenseState licenseState) { @@ -61,7 +64,7 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c final String refresh = request.param("refresh"); return channel -> new SecurityClient(client) - .prepareChangePassword(username, request.requiredContent(), request.getXContentType()) + .prepareChangePassword(username, request.requiredContent(), request.getXContentType(), passwordHasher) .setRefreshPolicy(refresh) .execute(new RestBuilderListener(channel) { @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java index 1b35e5684d91d..78229adfee962 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java @@ -16,8 +16,10 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.rest.RestRequestFilter; import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler; @@ -34,6 +36,8 @@ */ public class RestPutUserAction extends SecurityBaseRestHandler implements RestRequestFilter { + private final Hasher passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)); + public RestPutUserAction(Settings settings, RestController controller, XPackLicenseState licenseState) { super(settings, licenseState); controller.registerHandler(POST, "/_xpack/security/user/{username}", this); @@ -48,7 +52,7 @@ public String getName() { @Override public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { PutUserRequestBuilder requestBuilder = new SecurityClient(client) - .preparePutUser(request.param("username"), request.requiredContent(), request.getXContentType()) + .preparePutUser(request.param("username"), request.requiredContent(), request.getXContentType(), passwordHasher) .setRefreshPolicy(request.param("refresh")); return channel -> requestBuilder.execute(new RestBuilderListener(channel) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java index 31a1b1eaabfb0..246b584a83b01 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/AbstractPrivilegeTestCase.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.test.SecuritySingleNodeTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import java.io.IOException; @@ -33,8 +32,6 @@ */ public abstract class AbstractPrivilegeTestCase extends SecuritySingleNodeTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("passwd".toCharArray()))); - protected void assertAccessIsAllowed(String user, String method, String uri, String body, Map params) throws IOException { Response response = getRestClient().performRequest(method, uri, params, entityOrNull(body), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java index 2615365225f6e..8b9e195426493 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.Realm; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.core.security.user.User; @@ -42,7 +41,6 @@ import static org.hamcrest.Matchers.sameInstance; public class ClearRealmsCacheTests extends SecurityIntegTestCase { - private static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("passwd".toCharArray()))); private static String[] usernames; @@ -186,8 +184,10 @@ protected String configRoles() { @Override protected String configUsers() { StringBuilder builder = new StringBuilder(SecuritySettingsSource.CONFIG_STANDARD_USER); + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(new SecureString + ("passwd".toCharArray()))); for (String username : usernames) { - builder.append(username).append(":").append(USERS_PASSWD_HASHED).append("\n"); + builder.append(username).append(":").append(usersPasswdHashed).append("\n"); } return builder.toString(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java index daf3bba796330..03d0310a136b3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java @@ -8,7 +8,9 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -33,11 +35,6 @@ public class ClusterPrivilegeTests extends AbstractPrivilegeTestCase { " - names: 'someindex'\n" + " privileges: [ all ]\n"; - private static final String USERS = - "user_a:" + USERS_PASSWD_HASHED + "\n" + - "user_b:" + USERS_PASSWD_HASHED + "\n" + - "user_c:" + USERS_PASSWD_HASHED + "\n"; - private static final String USERS_ROLES = "role_a:user_a\n" + "role_b:user_b\n" + @@ -74,7 +71,13 @@ protected String configRoles() { @Override protected String configUsers() { - return super.configUsers() + USERS; + final String usersPasswdHashed = new String(Hasher.resolve( + randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")).hash(new SecureString("passwd".toCharArray()))); + return super.configUsers() + + "user_a:" + usersPasswdHashed + "\n" + + "user_b:" + usersPasswdHashed + "\n" + + "user_c:" + usersPasswdHashed + "\n"; + } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java index 016ccab87eb15..5ec9f8fbe06ad 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DateMathExpressionIntegTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.util.Collections; @@ -34,12 +33,13 @@ public class DateMathExpressionIntegTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(USERS_PASSWD)); @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); + return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n"; + "user1:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 5326954635e4c..b28b379494c42 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.util.Collections; import java.util.HashMap; @@ -40,16 +39,17 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(USERS_PASSWD)); @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); + return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n" + - "user2:" + USERS_PASSWD_HASHED + "\n" + - "user3:" + USERS_PASSWD_HASHED + "\n" + - "user4:" + USERS_PASSWD_HASHED + "\n" + - "user5:" + USERS_PASSWD_HASHED + "\n"; + "user1:" + usersPasswdHashed + "\n" + + "user2:" + usersPasswdHashed + "\n" + + "user3:" + usersPasswdHashed + "\n" + + "user4:" + usersPasswdHashed + "\n" + + "user5:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java index f4d1f429d8c6b..27a78981af069 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.test.SecurityIntegTestCase; import java.util.ArrayList; @@ -27,7 +26,6 @@ public class DocumentLevelSecurityRandomTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); // can't add a second test method, because each test run creates a new instance of this class and that will will result // in a new random value: @@ -35,9 +33,11 @@ public class DocumentLevelSecurityRandomTests extends SecurityIntegTestCase { @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); + StringBuilder builder = new StringBuilder(super.configUsers()); for (int i = 1; i <= numberOfRoles; i++) { - builder.append("user").append(i).append(':').append(USERS_PASSWD_HASHED).append('\n'); + builder.append("user").append(i).append(':').append(usersPasswdHashed).append('\n'); } return builder.toString(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index e02433c748e1e..2250facc37b51 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.security.LocalStateSecurity; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.util.Arrays; import java.util.Collection; @@ -81,7 +80,6 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(USERS_PASSWD)); @Override protected Collection> nodePlugins() { @@ -95,10 +93,11 @@ protected Collection> transportClientPlugins() { @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n" + - "user2:" + USERS_PASSWD_HASHED + "\n" + - "user3:" + USERS_PASSWD_HASHED + "\n" ; + "user1:" + usersPasswdHashed + "\n" + + "user2:" + usersPasswdHashed + "\n" + + "user3:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index 995b91d3628db..d1cdc1694a6f2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.test.SecurityIntegTestCase; import java.util.ArrayList; @@ -34,18 +33,19 @@ public class FieldLevelSecurityRandomTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); private static Set allowedFields; private static Set disAllowedFields; @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); + return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n" + - "user2:" + USERS_PASSWD_HASHED + "\n" + - "user3:" + USERS_PASSWD_HASHED + "\n" + - "user4:" + USERS_PASSWD_HASHED + "\n" ; + "user1:" + usersPasswdHashed + "\n" + + "user2:" + usersPasswdHashed + "\n" + + "user3:" + usersPasswdHashed + "\n" + + "user4:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 14840ed5594ff..d7d8c2ceb1be2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.security.LocalStateSecurity; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.util.Arrays; import java.util.Collection; @@ -67,7 +66,6 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); @Override protected Collection> nodePlugins() { @@ -82,15 +80,16 @@ protected Collection> transportClientPlugins() { @Override protected String configUsers() { + final String usersPasswHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n" + - "user2:" + USERS_PASSWD_HASHED + "\n" + - "user3:" + USERS_PASSWD_HASHED + "\n" + - "user4:" + USERS_PASSWD_HASHED + "\n" + - "user5:" + USERS_PASSWD_HASHED + "\n" + - "user6:" + USERS_PASSWD_HASHED + "\n" + - "user7:" + USERS_PASSWD_HASHED + "\n" + - "user8:" + USERS_PASSWD_HASHED + "\n"; + "user1:" + usersPasswHashed + "\n" + + "user2:" + usersPasswHashed + "\n" + + "user3:" + usersPasswHashed + "\n" + + "user4:" + usersPasswHashed + "\n" + + "user5:" + usersPasswHashed + "\n" + + "user6:" + usersPasswHashed + "\n" + + "user7:" + usersPasswHashed + "\n" + + "user8:" + usersPasswHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java index e1726c8c93176..825bfcd432f7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java @@ -8,6 +8,7 @@ import org.apache.http.message.BasicHeader; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.junit.Before; @@ -84,22 +85,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase { " privileges: [ index ]\n" + "\n"; - private static final String USERS = - "admin:" + USERS_PASSWD_HASHED + "\n" + - "u1:" + USERS_PASSWD_HASHED + "\n" + - "u2:" + USERS_PASSWD_HASHED + "\n" + - "u3:" + USERS_PASSWD_HASHED + "\n" + - "u4:" + USERS_PASSWD_HASHED + "\n" + - "u5:" + USERS_PASSWD_HASHED + "\n" + - "u6:" + USERS_PASSWD_HASHED + "\n" + - "u7:" + USERS_PASSWD_HASHED + "\n"+ - "u8:" + USERS_PASSWD_HASHED + "\n"+ - "u9:" + USERS_PASSWD_HASHED + "\n" + - "u11:" + USERS_PASSWD_HASHED + "\n" + - "u12:" + USERS_PASSWD_HASHED + "\n" + - "u13:" + USERS_PASSWD_HASHED + "\n" + - "u14:" + USERS_PASSWD_HASHED + "\n"; - private static final String USERS_ROLES = "all_indices_role:admin,u8\n" + "all_cluster_role:admin\n" + @@ -129,7 +114,24 @@ protected String configRoles() { @Override protected String configUsers() { - return super.configUsers() + USERS; + final String usersPasswdHashed = new String(Hasher.resolve( + randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")).hash(new SecureString("passwd".toCharArray()))); + + return super.configUsers() + + "admin:" + usersPasswdHashed + "\n" + + "u1:" + usersPasswdHashed + "\n" + + "u2:" + usersPasswdHashed + "\n" + + "u3:" + usersPasswdHashed + "\n" + + "u4:" + usersPasswdHashed + "\n" + + "u5:" + usersPasswdHashed + "\n" + + "u6:" + usersPasswdHashed + "\n" + + "u7:" + usersPasswdHashed + "\n" + + "u8:" + usersPasswdHashed + "\n" + + "u9:" + usersPasswdHashed + "\n" + + "u11:" + usersPasswdHashed + "\n" + + "u12:" + usersPasswdHashed + "\n" + + "u13:" + usersPasswdHashed + "\n" + + "u14:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java index 9982b42b859f1..b20c1adb9b997 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.test.SecurityIntegTestCase; import java.util.Collections; @@ -24,12 +23,12 @@ public class IndicesPermissionsWithAliasesWildcardsAndRegexsTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return super.configUsers() + - "user1:" + USERS_PASSWD_HASHED + "\n"; + "user1:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index a3174a02e99dd..cc080a846fae3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import java.util.Locale; @@ -40,7 +39,6 @@ public class KibanaUserRoleIntegTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("change_me".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); @Override public String configRoles() { @@ -55,8 +53,9 @@ public String configRoles() { @Override public String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return super.configUsers() + - "kibana_user:" + USERS_PASSWD_HASHED; + "kibana_user:" + usersPasswdHashed; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index a8750d5b80232..675300e25760e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; @@ -28,8 +27,8 @@ import static org.hamcrest.Matchers.is; public class MultipleIndicesPermissionsTests extends SecurityIntegTestCase { - protected static final SecureString PASSWD = new SecureString("passwd".toCharArray()); - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(PASSWD)); + + protected static final SecureString USERS_PASSWD = new SecureString("passwd".toCharArray()); @Override protected String configRoles() { @@ -58,9 +57,10 @@ protected String configRoles() { @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return SecuritySettingsSource.CONFIG_STANDARD_USER + - "user_a:" + USERS_PASSWD_HASHED + "\n" + - "user_ab:" + USERS_PASSWD_HASHED + "\n"; + "user_a:" + usersPasswdHashed + "\n" + + "user_ab:" + usersPasswdHashed + "\n"; } @Override @@ -145,7 +145,7 @@ public void testMultipleRoles() throws Exception { Client client = internalCluster().transportClient(); SearchResponse response = client - .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", PASSWD))) + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", USERS_PASSWD))) .prepareSearch("a") .get(); assertNoFailures(response); @@ -156,7 +156,7 @@ public void testMultipleRoles() throws Exception { new String[] { "*" } : new String[] {}; response = client - .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", PASSWD))) + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", USERS_PASSWD))) .prepareSearch(indices) .get(); assertNoFailures(response); @@ -165,7 +165,7 @@ public void testMultipleRoles() throws Exception { try { indices = randomBoolean() ? new String[] { "a", "b" } : new String[] { "b", "a" }; client - .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", PASSWD))) + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_a", USERS_PASSWD))) .prepareSearch(indices) .get(); fail("expected an authorization excpetion when trying to search on multiple indices where there are no search permissions on " + @@ -175,14 +175,14 @@ public void testMultipleRoles() throws Exception { assertThat(e.status(), is(RestStatus.FORBIDDEN)); } - response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", PASSWD))) + response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", USERS_PASSWD))) .prepareSearch("b") .get(); assertNoFailures(response); assertHitCount(response, 1); indices = randomBoolean() ? new String[] { "a", "b" } : new String[] { "b", "a" }; - response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", PASSWD))) + response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", USERS_PASSWD))) .prepareSearch(indices) .get(); assertNoFailures(response); @@ -192,7 +192,7 @@ public void testMultipleRoles() throws Exception { new String[] { "_all"} : randomBoolean() ? new String[] { "*" } : new String[] {}; - response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", PASSWD))) + response = client.filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_ab", USERS_PASSWD))) .prepareSearch(indices) .get(); assertNoFailures(response); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java index 41b5787cb26f3..f3fea81e20127 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import java.util.Collections; @@ -33,7 +32,6 @@ * index template actions. */ public class PermissionPrecedenceTests extends SecurityIntegTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("test123".toCharArray()))); @Override protected String configRoles() { @@ -51,9 +49,10 @@ protected String configRoles() { @Override protected String configUsers() { - return "admin:" + USERS_PASSWD_HASHED + "\n" + - "client:" + USERS_PASSWD_HASHED + "\n" + - "user:" + USERS_PASSWD_HASHED + "\n"; + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(new SecureString("test123".toCharArray()))); + return "admin:" + usersPasswdHashed + "\n" + + "client:" + usersPasswdHashed + "\n" + + "user:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java index 2ec899dbafe0d..9f86887566ac4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.test.SecurityIntegTestCase; import org.junit.After; import org.junit.Before; @@ -33,15 +32,15 @@ import static org.hamcrest.Matchers.is; public class SecurityClearScrollTests extends SecurityIntegTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("change_me".toCharArray()))); private List scrollIds; @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(new SecureString("change_me".toCharArray()))); return super.configUsers() + - "allowed_user:" + USERS_PASSWD_HASHED + "\n" + - "denied_user:" + USERS_PASSWD_HASHED + "\n" ; + "allowed_user:" + usersPasswdHashed + "\n" + + "denied_user:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java index 2297a5353b6e0..f30fe5f8ec81a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java @@ -86,11 +86,6 @@ public class LicensingTests extends SecurityIntegTestCase { " - names: 'b'\n" + " privileges: [all]\n"; - public static final String USERS = - SecuritySettingsSource.CONFIG_STANDARD_USER + - "user_a:{plain}passwd\n" + - "user_b:{plain}passwd\n"; - public static final String USERS_ROLES = SecuritySettingsSource.CONFIG_STANDARD_USER_ROLES + "role_a:user_a,user_b\n" + @@ -103,7 +98,9 @@ protected String configRoles() { @Override protected String configUsers() { - return USERS; + return SecuritySettingsSource.CONFIG_STANDARD_USER + + "user_a:{plain}passwd\n" + + "user_b:{plain}passwd\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 815f26942767a..97dd7866dc006 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.core.XPackClient; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.SecurityField; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; import org.elasticsearch.xpack.security.LocalStateSecurity; @@ -88,7 +89,6 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase { */ private static CustomSecuritySettingsSource customSecuritySettingsSource = null; - @BeforeClass public static void generateBootstrapPassword() { BOOTSTRAP_PASSWORD = TEST_PASSWORD_SECURE_STRING.clone(); @@ -522,4 +522,8 @@ private static Index resolveSecurityIndex(MetaData metaData) { protected boolean isTransportSSLEnabled() { return customSecuritySettingsSource.isSslEnabled(); } + + protected static Hasher getFastStoredHashAlgoForTests() { + return Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 8ad1c61029a97..cc8c61a5c32e4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -42,6 +42,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; import static org.apache.lucene.util.LuceneTestCase.createTempFile; +import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile; @@ -57,7 +58,8 @@ public class SecuritySettingsSource extends ClusterDiscoveryConfiguration.Unicas public static final String TEST_USER_NAME = "test_user"; public static final String TEST_PASSWORD_HASHED = - new String(Hasher.BCRYPT.hash(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))); + new String(Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt9", "bcrypt8", "bcrypt")). + hash(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))); public static final String TEST_ROLE = "user"; public static final String TEST_SUPERUSER = "test_superuser"; @@ -133,7 +135,7 @@ public Settings nodeSettings(int nodeOrdinal) { .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE) .put("xpack.security.authc.realms.file.order", 0) .put("xpack.security.authc.realms.index.type", NativeRealmSettings.TYPE) - .put("xpack.security.authc.realms.index.order", "1"); + .put("xpack.security.authc.realms.index.order", "1"); addNodeSSLSettings(builder); return builder.build(); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheckTests.java new file mode 100644 index 0000000000000..8ca5c6c7216c5 --- /dev/null +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PasswordHashingAlgorithmBootstrapCheckTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security; + +import org.elasticsearch.bootstrap.BootstrapContext; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackSettings; + +import javax.crypto.SecretKeyFactory; +import java.security.NoSuchAlgorithmException; + +public class PasswordHashingAlgorithmBootstrapCheckTests extends ESTestCase { + + public void testPasswordHashingAlgorithmBootstrapCheck() { + Settings settings = Settings.EMPTY; + assertFalse(new PasswordHashingAlgorithmBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); + // The following two will always pass because for now we only test in environments where PBKDF2WithHMACSHA512 is supported + assertTrue(isSecretkeyFactoryAlgoAvailable("PBKDF2WithHMACSHA512")); + settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2_10000").build(); + assertFalse(new PasswordHashingAlgorithmBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); + + settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "PBKDF2").build(); + assertFalse(new PasswordHashingAlgorithmBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); + + settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT").build(); + assertFalse(new PasswordHashingAlgorithmBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); + + settings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), "BCRYPT11").build(); + assertFalse(new PasswordHashingAlgorithmBootstrapCheck().check(new BootstrapContext(settings, null)).isFailure()); + } + + private boolean isSecretkeyFactoryAlgoAvailable(String algorithmId) { + try { + SecretKeyFactory.getInstance(algorithmId); + return true; + } catch (NoSuchAlgorithmException e) { + return false; + } + } +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java index ac3308e2cc35d..a7bad498bdc33 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.action.user.PutUserRequest; import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -35,7 +36,7 @@ public void testNullValuesForEmailAndFullName() throws IOException { "}"; PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); - builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT); PutUserRequest request = builder.request(); assertThat(request.username(), is("kibana4")); @@ -55,7 +56,7 @@ public void testMissingEmailFullName() throws Exception { "}"; PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); - builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT); PutUserRequest request = builder.request(); assertThat(request.username(), is("kibana4")); @@ -76,7 +77,7 @@ public void testWithFullNameAndEmail() throws IOException { "}"; PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); - builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON); + builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT); PutUserRequest request = builder.request(); assertThat(request.username(), is("kibana4")); @@ -98,7 +99,7 @@ public void testInvalidFullname() throws IOException { PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, - () -> builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)); + () -> builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT)); assertThat(e.getMessage(), containsString("expected field [full_name] to be of type string")); } @@ -114,7 +115,7 @@ public void testInvalidEmail() throws IOException { PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, - () -> builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)); + () -> builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT)); assertThat(e.getMessage(), containsString("expected field [email] to be of type string")); } @@ -131,7 +132,7 @@ public void testWithEnabled() throws IOException { PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class)); PutUserRequest request = - builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON).request(); + builder.source("kibana4", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, Hasher.BCRYPT).request(); assertFalse(request.enabled()); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java index 8808ab92a41f9..0652de8891d0f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequest; import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse; import org.elasticsearch.xpack.core.security.authc.support.Hasher; @@ -52,11 +53,12 @@ public void testAnonymousUser() { TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportChangePasswordAction action = new TransportChangePasswordAction(settings, transportService, - mock(ActionFilters.class), usersStore); + mock(ActionFilters.class), usersStore); ChangePasswordRequest request = new ChangePasswordRequest(); request.username(anonymousUser.principal()); - request.passwordHash(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + request.passwordHash(Hasher.resolve( + randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); final AtomicReference throwableRef = new AtomicReference<>(); final AtomicReference responseRef = new AtomicReference<>(); @@ -83,11 +85,12 @@ public void testInternalUsers() { TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportChangePasswordAction action = new TransportChangePasswordAction(Settings.EMPTY, transportService, - mock(ActionFilters.class), usersStore); + mock(ActionFilters.class), usersStore); ChangePasswordRequest request = new ChangePasswordRequest(); request.username(randomFrom(SystemUser.INSTANCE.principal(), XPackUser.INSTANCE.principal())); - request.passwordHash(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + request.passwordHash(Hasher.resolve( + randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); final AtomicReference throwableRef = new AtomicReference<>(); final AtomicReference responseRef = new AtomicReference<>(); @@ -110,11 +113,13 @@ public void onFailure(Exception e) { } public void testValidUser() { + final String hashingAlgorithm = randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9"); final User user = randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe")); NativeUsersStore usersStore = mock(NativeUsersStore.class); + final Hasher hasher = Hasher.resolve(hashingAlgorithm); ChangePasswordRequest request = new ChangePasswordRequest(); request.username(user.principal()); - request.passwordHash(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + request.passwordHash(hasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); doAnswer(invocation -> { Object[] args = invocation.getArguments(); assert args.length == 2; @@ -124,9 +129,10 @@ public void testValidUser() { }).when(usersStore).changePassword(eq(request), any(ActionListener.class)); TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); - TransportChangePasswordAction action = new TransportChangePasswordAction(Settings.EMPTY, transportService, - mock(ActionFilters.class), usersStore); - + Settings passwordHashingSettings = Settings.builder(). + put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hashingAlgorithm).build(); + TransportChangePasswordAction action = new TransportChangePasswordAction(passwordHashingSettings, transportService, + mock(ActionFilters.class), usersStore); final AtomicReference throwableRef = new AtomicReference<>(); final AtomicReference responseRef = new AtomicReference<>(); action.doExecute(mock(Task.class), request, new ActionListener() { @@ -147,12 +153,46 @@ public void onFailure(Exception e) { verify(usersStore, times(1)).changePassword(eq(request), any(ActionListener.class)); } + public void testIncorrectPasswordHashingAlgorithm() { + final User user = randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe")); + final Hasher hasher = Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt9", "bcrypt5")); + NativeUsersStore usersStore = mock(NativeUsersStore.class); + ChangePasswordRequest request = new ChangePasswordRequest(); + request.username(user.principal()); + request.passwordHash(hasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + final AtomicReference throwableRef = new AtomicReference<>(); + final AtomicReference responseRef = new AtomicReference<>(); + TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, + x -> null, null, Collections.emptySet()); + Settings passwordHashingSettings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), + randomFrom("pbkdf2_50000", "pbkdf2_10000", "bcrypt11", "bcrypt8", "bcrypt")).build(); + TransportChangePasswordAction action = new TransportChangePasswordAction(passwordHashingSettings, transportService, + mock(ActionFilters.class), usersStore); + action.doExecute(mock(Task.class), request, new ActionListener() { + @Override + public void onResponse(ChangePasswordResponse changePasswordResponse) { + responseRef.set(changePasswordResponse); + } + + @Override + public void onFailure(Exception e) { + throwableRef.set(e); + } + }); + + assertThat(responseRef.get(), is(nullValue())); + assertThat(throwableRef.get(), instanceOf(IllegalArgumentException.class)); + assertThat(throwableRef.get().getMessage(), containsString("incorrect password hashing algorithm")); + verifyZeroInteractions(usersStore); + } + public void testException() { + final String hashingAlgorithm = randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9"); final User user = randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe")); NativeUsersStore usersStore = mock(NativeUsersStore.class); ChangePasswordRequest request = new ChangePasswordRequest(); request.username(user.principal()); - request.passwordHash(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + request.passwordHash(Hasher.resolve(hashingAlgorithm).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); final Exception e = randomFrom(new ElasticsearchSecurityException(""), new IllegalStateException(), new RuntimeException()); doAnswer(new Answer() { public Void answer(InvocationOnMock invocation) { @@ -165,8 +205,10 @@ public Void answer(InvocationOnMock invocation) { }).when(usersStore).changePassword(eq(request), any(ActionListener.class)); TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); - TransportChangePasswordAction action = new TransportChangePasswordAction(Settings.EMPTY, transportService, - mock(ActionFilters.class), usersStore); + Settings passwordHashingSettings = Settings.builder(). + put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), hashingAlgorithm).build(); + TransportChangePasswordAction action = new TransportChangePasswordAction(passwordHashingSettings, transportService, + mock(ActionFilters.class), usersStore); final AtomicReference throwableRef = new AtomicReference<>(); final AtomicReference responseRef = new AtomicReference<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java index 86a70bdf7e08e..99e35f1b2145e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java @@ -165,7 +165,8 @@ public void testValidUser() { final PutUserRequest request = new PutUserRequest(); request.username(user.principal()); if (isCreate) { - request.passwordHash(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); + request.passwordHash(Hasher.resolve( + randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")).hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)); } final boolean created = isCreate ? randomBoolean() : false; // updates should always return false for create doAnswer(new Answer() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmSettingsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmSettingsTests.java index 3871574b76c51..b177d17793f89 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmSettingsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RealmSettingsTests.java @@ -27,8 +27,7 @@ import static org.hamcrest.Matchers.notNullValue; public class RealmSettingsTests extends ESTestCase { - - private static final List HASH_ALGOS = Arrays.stream(Hasher.values()).map(Hasher::name).collect(Collectors.toList()); + private static final List CACHE_HASHING_ALGOS = Arrays.stream(Hasher.values()).map(Hasher::name).collect(Collectors.toList()); public void testRealmWithoutTypeDoesNotValidate() throws Exception { final Settings.Builder builder = baseSettings("x", false); @@ -260,8 +259,8 @@ private Settings.Builder baseSettings(String type, boolean withCacheSettings) { .put("enabled", true); if (withCacheSettings) { builder.put("cache.ttl", randomPositiveTimeValue()) - .put("cache.max_users", randomIntBetween(1_000, 1_000_000)) - .put("cache.hash_algo", randomFrom(HASH_ALGOS)); + .put("cache.max_users", randomIntBetween(1_000, 1_000_000)) + .put("cache.hash_algo", randomFrom(CACHE_HASHING_ALGOS)); } return builder; } @@ -330,4 +329,4 @@ private Setting group() { assertThat(list, hasSize(1)); return list.get(0); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index 58cb515081a59..da48136e0a3d3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -77,7 +77,7 @@ public void testRetrieveUsers() throws Exception { Set addedUsers = new HashSet(numToAdd); for (int i = 0; i < numToAdd; i++) { String uname = randomAlphaOfLength(5); - c.preparePutUser(uname, "s3kirt".toCharArray(), "role1", "user").get(); + c.preparePutUser(uname, "s3kirt".toCharArray(), getFastStoredHashAlgoForTests(), "role1", "user").get(); addedUsers.add(uname); } logger.error("--> waiting for .security index"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index a0550b4c1ce15..62e0d43e87fc7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -111,7 +111,7 @@ public void setupAnonymousRoleIfNecessary() throws Exception { public void testDeletingNonexistingUserAndRole() throws Exception { SecurityClient c = securityClient(); // first create the index so it exists - c.preparePutUser("joe", "s3kirt".toCharArray(), "role1", "user").get(); + c.preparePutUser("joe", "s3kirt".toCharArray(), getFastStoredHashAlgoForTests(), "role1", "user").get(); DeleteUserResponse resp = c.prepareDeleteUser("missing").get(); assertFalse("user shouldn't be found", resp.found()); DeleteRoleResponse resp2 = c.prepareDeleteRole("role").get(); @@ -131,7 +131,7 @@ public void testAddAndGetUser() throws Exception { final List existingUsers = Arrays.asList(c.prepareGetUsers().get().users()); final int existing = existingUsers.size(); logger.error("--> creating user"); - c.preparePutUser("joe", "s3kirt".toCharArray(), "role1", "user").get(); + c.preparePutUser("joe", "s3kirt".toCharArray(), getFastStoredHashAlgoForTests(), "role1", "user").get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); logger.info("--> retrieving user"); @@ -142,8 +142,8 @@ public void testAddAndGetUser() throws Exception { assertArrayEquals(joe.roles(), new String[]{"role1", "user"}); logger.info("--> adding two more users"); - c.preparePutUser("joe2", "s3kirt2".toCharArray(), "role2", "user").get(); - c.preparePutUser("joe3", "s3kirt3".toCharArray(), "role3", "user").get(); + c.preparePutUser("joe2", "s3kirt2".toCharArray(), getFastStoredHashAlgoForTests(), "role2", "user").get(); + c.preparePutUser("joe3", "s3kirt3".toCharArray(), getFastStoredHashAlgoForTests(), "role3", "user").get(); GetUsersResponse allUsersResp = c.prepareGetUsers().get(); assertTrue("users should exist", allUsersResp.hasUsers()); assertEquals("should be " + (3 + existing) + " users total", 3 + existing, allUsersResp.users().length); @@ -237,7 +237,7 @@ public void testAddUserAndRoleThenAuth() throws Exception { new BytesArray("{\"match_all\": {}}")) .get(); logger.error("--> creating user"); - c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get(); + c.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), "test_role").get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); logger.info("--> retrieving user"); @@ -252,13 +252,13 @@ public void testAddUserAndRoleThenAuth() throws Exception { String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get(); - assertEquals(searchResp.getHits().getTotalHits(), 1L); + assertEquals(1L, searchResp.getHits().getTotalHits()); } public void testUpdatingUserAndAuthentication() throws Exception { SecurityClient c = securityClient(); logger.error("--> creating user"); - c.preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + c.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); logger.info("--> retrieving user"); @@ -273,9 +273,9 @@ public void testUpdatingUserAndAuthentication() throws Exception { String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get(); - assertEquals(searchResp.getHits().getTotalHits(), 1L); + assertEquals(1L, searchResp.getHits().getTotalHits()); - c.preparePutUser("joe", "s3krit2".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + c.preparePutUser("joe", "s3krit2".toCharArray(), getFastStoredHashAlgoForTests(), SecuritySettingsSource.TEST_ROLE).get(); try { client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get(); @@ -287,13 +287,14 @@ public void testUpdatingUserAndAuthentication() throws Exception { token = basicAuthHeaderValue("joe", new SecureString("s3krit2".toCharArray())); searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get(); - assertEquals(searchResp.getHits().getTotalHits(), 1L); + assertEquals(1L, searchResp.getHits().getTotalHits()); } public void testCreateDeleteAuthenticate() { SecurityClient c = securityClient(); logger.error("--> creating user"); - c.preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + c.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), + SecuritySettingsSource.TEST_ROLE).get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); logger.info("--> retrieving user"); @@ -308,7 +309,7 @@ public void testCreateDeleteAuthenticate() { String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get(); - assertEquals(searchResp.getHits().getTotalHits(), 1L); + assertEquals(1L, searchResp.getHits().getTotalHits()); DeleteUserResponse response = c.prepareDeleteUser("joe").get(); assertThat(response.found(), is(true)); @@ -331,7 +332,7 @@ public void testCreateAndUpdateRole() { new BytesArray("{\"match_all\": {}}")) .get(); logger.error("--> creating user"); - c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get(); + c.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), "test_role").get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); @@ -380,7 +381,7 @@ public void testAuthenticateWithDeletedRole() { .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null, new BytesArray("{\"match_all\": {}}")) .get(); - c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get(); + c.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), "test_role").get(); logger.error("--> waiting for .security index"); ensureGreen(SECURITY_INDEX_NAME); @@ -414,7 +415,7 @@ public void testPutUserWithoutPassword() { assertThat(client.prepareGetUsers("joes").get().hasUsers(), is(false)); // check that putting a user without a password fails if the user doesn't exist try { - client.preparePutUser("joe", null, "admin_role").get(); + client.preparePutUser("joe", null, getFastStoredHashAlgoForTests(), "admin_role").get(); fail("cannot create a user without a password"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("password must be specified")); @@ -423,7 +424,8 @@ public void testPutUserWithoutPassword() { assertThat(client.prepareGetUsers("joes").get().hasUsers(), is(false)); // create joe with a password and verify the user works - client.preparePutUser("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), "admin_role").get(); + client.preparePutUser("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), + getFastStoredHashAlgoForTests(), "admin_role").get(); assertThat(client.prepareGetUsers("joe").get().hasUsers(), is(true)); final String token = basicAuthHeaderValue("joe", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster() @@ -431,7 +433,7 @@ public void testPutUserWithoutPassword() { assertFalse(response.isTimedOut()); // modify joe without sending the password - client.preparePutUser("joe", null, "read_role").fullName("Joe Smith").get(); + client.preparePutUser("joe", null, getFastStoredHashAlgoForTests(), "read_role").fullName("Joe Smith").get(); GetUsersResponse getUsersResponse = client.prepareGetUsers("joe").get(); assertThat(getUsersResponse.hasUsers(), is(true)); assertThat(getUsersResponse.users().length, is(1)); @@ -452,7 +454,8 @@ public void testPutUserWithoutPassword() { // update the user with password and admin role again String secondPassword = SecuritySettingsSourceField.TEST_PASSWORD + "2"; - client.preparePutUser("joe", secondPassword.toCharArray(), "admin_role").fullName("Joe Smith").get(); + client.preparePutUser("joe", secondPassword.toCharArray(), getFastStoredHashAlgoForTests(), "admin_role"). + fullName("Joe Smith").get(); getUsersResponse = client.prepareGetUsers("joe").get(); assertThat(getUsersResponse.hasUsers(), is(true)); assertThat(getUsersResponse.users().length, is(1)); @@ -480,7 +483,8 @@ public void testPutUserWithoutPassword() { public void testCannotCreateUserWithShortPassword() throws Exception { SecurityClient client = securityClient(); try { - client.preparePutUser("joe", randomAlphaOfLengthBetween(0, 5).toCharArray(), "admin_role").get(); + client.preparePutUser("joe", randomAlphaOfLengthBetween(0, 5).toCharArray(), getFastStoredHashAlgoForTests(), + "admin_role").get(); fail("cannot create a user without a password < 6 characters"); } catch (ValidationException v) { assertThat(v.getMessage().contains("password"), is(true)); @@ -490,7 +494,8 @@ public void testCannotCreateUserWithShortPassword() throws Exception { public void testCannotCreateUserWithInvalidCharactersInName() throws Exception { SecurityClient client = securityClient(); ValidationException v = expectThrows(ValidationException.class, - () -> client.preparePutUser("fóóbár", "my-am@zing-password".toCharArray(), "admin_role").get() + () -> client.preparePutUser("fóóbár", "my-am@zing-password".toCharArray(), getFastStoredHashAlgoForTests(), + "admin_role").get() ); assertThat(v.getMessage(), containsString("names must be")); } @@ -500,7 +505,8 @@ public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception { SecurityClient client = securityClient(); if (randomBoolean()) { - client.preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + client.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), + SecuritySettingsSource.TEST_ROLE).get(); } else { client.preparePutRole("read_role") .cluster("none") @@ -520,7 +526,7 @@ public void testOperationsOnReservedUsers() throws Exception { final String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> securityClient().preparePutUser(username, randomBoolean() ? SecuritySettingsSourceField.TEST_PASSWORD.toCharArray() - : null, "admin").get()); + : null, getFastStoredHashAlgoForTests(), "admin").get()); assertThat(exception.getMessage(), containsString("Username [" + username + "] is reserved")); exception = expectThrows(IllegalArgumentException.class, @@ -532,19 +538,22 @@ public void testOperationsOnReservedUsers() throws Exception { assertThat(exception.getMessage(), containsString("user [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is anonymous")); exception = expectThrows(IllegalArgumentException.class, - () -> securityClient().prepareChangePassword(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray()).get()); + () -> securityClient().prepareChangePassword(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray(), + getFastStoredHashAlgoForTests()).get()); assertThat(exception.getMessage(), containsString("user [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is anonymous")); exception = expectThrows(IllegalArgumentException.class, - () -> securityClient().preparePutUser(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray()).get()); + () -> securityClient().preparePutUser(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray(), + getFastStoredHashAlgoForTests()).get()); assertThat(exception.getMessage(), containsString("Username [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is reserved")); exception = expectThrows(IllegalArgumentException.class, - () -> securityClient().preparePutUser(SystemUser.NAME, "foobar".toCharArray()).get()); + () -> securityClient().preparePutUser(SystemUser.NAME, "foobar".toCharArray(), getFastStoredHashAlgoForTests()).get()); assertThat(exception.getMessage(), containsString("user [" + SystemUser.NAME + "] is internal")); exception = expectThrows(IllegalArgumentException.class, - () -> securityClient().prepareChangePassword(SystemUser.NAME, "foobar".toCharArray()).get()); + () -> securityClient().prepareChangePassword(SystemUser.NAME, "foobar".toCharArray(), + getFastStoredHashAlgoForTests()).get()); assertThat(exception.getMessage(), containsString("user [" + SystemUser.NAME + "] is internal")); exception = expectThrows(IllegalArgumentException.class, @@ -582,7 +591,8 @@ public void testOperationsOnReservedRoles() throws Exception { } public void testCreateAndChangePassword() throws Exception { - securityClient().preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + securityClient().preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), + SecuritySettingsSource.TEST_ROLE).get(); final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)) .admin().cluster().prepareHealth().get(); @@ -590,8 +600,7 @@ public void testCreateAndChangePassword() throws Exception { ChangePasswordResponse passwordResponse = securityClient( client().filterWithHeader(Collections.singletonMap("Authorization", token))) - .prepareChangePassword("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()) - .get(); + .prepareChangePassword("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), getFastStoredHashAlgoForTests()).get(); assertThat(passwordResponse, notNullValue()); @@ -671,7 +680,8 @@ public void testRealmUsageStats() { final int numNativeUsers = scaledRandomIntBetween(1, 32); SecurityClient securityClient = new SecurityClient(client()); for (int i = 0; i < numNativeUsers; i++) { - securityClient.preparePutUser("joe" + i, "s3krit".toCharArray(), "superuser").get(); + securityClient.preparePutUser("joe" + i, "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), + "superuser").get(); } XPackUsageResponse response = new XPackUsageRequestBuilder(client()).get(); @@ -690,7 +700,9 @@ public void testRealmUsageStats() { } public void testSetEnabled() throws Exception { - securityClient().preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); + + securityClient().preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), + SecuritySettingsSource.TEST_ROLE).get(); final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray())); ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)) .admin().cluster().prepareHealth().get(); @@ -714,7 +726,7 @@ public void testSetEnabled() throws Exception { public void testNegativeLookupsThenCreateRole() throws Exception { SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser("joe", "s3krit".toCharArray(), "unknown_role").get(); + securityClient.preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), "unknown_role").get(); final int negativeLookups = scaledRandomIntBetween(1, 10); for (int i = 0; i < negativeLookups; i++) { @@ -750,8 +762,9 @@ public void testNegativeLookupsThenCreateRole() throws Exception { * the loader returned a null value, while the other caller(s) would get a null value unexpectedly */ public void testConcurrentRunAs() throws Exception { - securityClient().preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.TEST_ROLE).get(); - securityClient().preparePutUser("executor", "s3krit".toCharArray(), "superuser").get(); + securityClient().preparePutUser("joe", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), SecuritySettingsSource + .TEST_ROLE).get(); + securityClient().preparePutUser("executor", "s3krit".toCharArray(), getFastStoredHashAlgoForTests(), "superuser").get(); final String token = basicAuthHeaderValue("executor", new SecureString("s3krit".toCharArray())); final Client client = client().filterWithHeader(MapBuilder.newMapBuilder() .put("Authorization", token) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 9fbcaa493dd96..d8af70cb6bec7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -121,7 +121,7 @@ public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception { final NativeUsersStore.ReservedUserInfo userInfo = future.get(); assertThat(userInfo.hasEmptyPassword, equalTo(true)); assertThat(userInfo.enabled, equalTo(true)); - assertThat(userInfo.passwordHash, equalTo(ReservedRealm.EMPTY_PASSWORD_HASH)); + assertTrue(Hasher.verifyHash(new SecureString("".toCharArray()), userInfo.passwordHash)); } public void testVerifyUserWithCorrectPassword() throws Exception { @@ -207,6 +207,7 @@ private Tuple> find } private void respondToGetUserRequest(String username, SecureString password, String[] roles) throws IOException { + // Native users store is initiated with default hashing algorithm final Map values = new HashMap<>(); values.put(User.Fields.USERNAME.getPreferredName(), username); values.put(User.Fields.PASSWORD.getPreferredName(), String.valueOf(Hasher.BCRYPT.hash(password))); @@ -241,4 +242,4 @@ private NativeUsersStore startNativeUsersStore() { return new NativeUsersStore(Settings.EMPTY, client, securityIndex); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java index 2ec7ed7ea2204..c0f69fc1c52bb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java @@ -76,7 +76,7 @@ public void testChangingPassword() { } ChangePasswordResponse response = securityClient() - .prepareChangePassword(username, Arrays.copyOf(newPassword, newPassword.length)) + .prepareChangePassword(username, Arrays.copyOf(newPassword, newPassword.length), getFastStoredHashAlgoForTests()) .get(); assertThat(response, notNullValue()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java index b483595f8ec20..fad3d43e6d5f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java @@ -77,12 +77,21 @@ public void setupMocks() throws Exception { when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); } + public void testInvalidHashingAlgorithmFails() { + final String invalidAlgoId = randomFrom("sha1", "md5", "noop"); + final Settings invalidSettings = Settings.builder().put("xpack.security.authc.password_hashing.algorithm", invalidAlgoId).build(); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new ReservedRealm(mock(Environment.class), + invalidSettings, usersStore, new AnonymousUser(Settings.EMPTY), securityIndex, threadPool)); + assertThat(exception.getMessage(), containsString(invalidAlgoId)); + assertThat(exception.getMessage(), containsString("Only pbkdf2 or bcrypt family algorithms can be used for password hashing")); + } + public void testReservedUserEmptyPasswordAuthenticationFails() throws Throwable { final String principal = randomFrom(UsernamesField.ELASTIC_NAME, UsernamesField.KIBANA_NAME, UsernamesField.LOGSTASH_NAME, - UsernamesField.BEATS_NAME); + UsernamesField.BEATS_NAME); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); @@ -97,8 +106,8 @@ public void testAuthenticationDisabled() throws Throwable { when(securityIndex.indexExists()).thenReturn(true); } final ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(settings), securityIndex, threadPool); + new ReservedRealm(mock(Environment.class), settings, usersStore, + new AnonymousUser(settings), securityIndex, threadPool); final User expected = randomReservedUser(true); final String principal = expected.principal(); @@ -120,14 +129,16 @@ public void testAuthenticationDisabledUserWithStoredPassword() throws Throwable private void verifySuccessfulAuthentication(boolean enabled) throws Exception { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); final User expectedUser = randomReservedUser(enabled); final String principal = expectedUser.principal(); final SecureString newPassword = new SecureString("foobar".toCharArray()); + // Mocked users store is initiated with default hashing algorithm + final Hasher hasher = Hasher.resolve("bcrypt"); when(securityIndex.indexExists()).thenReturn(true); doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[1]; - callback.onResponse(new ReservedUserInfo(Hasher.BCRYPT.hash(newPassword), enabled, false)); + callback.onResponse(new ReservedUserInfo(hasher.hash(newPassword), enabled, false)); return null; }).when(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class)); @@ -139,7 +150,7 @@ private void verifySuccessfulAuthentication(boolean enabled) throws Exception { // the realm assumes it owns the hashed password so it fills it with 0's doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[1]; - callback.onResponse(new ReservedUserInfo(Hasher.BCRYPT.hash(newPassword), true, false)); + callback.onResponse(new ReservedUserInfo(hasher.hash(newPassword), true, false)); return null; }).when(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class)); @@ -160,8 +171,8 @@ private void verifySuccessfulAuthentication(boolean enabled) throws Exception { public void testLookup() throws Exception { final ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); @@ -185,8 +196,8 @@ public void testLookup() throws Exception { public void testLookupDisabled() throws Exception { Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build(); final ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), - securityIndex, threadPool); + new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), + securityIndex, threadPool); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); @@ -199,8 +210,8 @@ public void testLookupDisabled() throws Exception { public void testLookupThrows() throws Exception { final ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); when(securityIndex.indexExists()).thenReturn(true); @@ -247,22 +258,22 @@ public void testIsReservedDisabled() { public void testGetUsers() { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); assertThat(userFuture.actionGet(), - containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true), new BeatsSystemUser(true))); + containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true), new BeatsSystemUser(true))); } public void testGetUsersDisabled() { final boolean anonymousEnabled = randomBoolean(); Settings settings = Settings.builder() - .put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false) - .put(AnonymousUser.ROLES_SETTING.getKey(), anonymousEnabled ? "user" : "") - .build(); + .put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false) + .put(AnonymousUser.ROLES_SETTING.getKey(), anonymousEnabled ? "user" : "") + .build(); final AnonymousUser anonymousUser = new AnonymousUser(settings); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, - securityIndex, threadPool); + securityIndex, threadPool); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); if (anonymousEnabled) { @@ -275,11 +286,13 @@ public void testGetUsersDisabled() { public void testFailedAuthentication() throws Exception { when(securityIndex.indexExists()).thenReturn(true); SecureString password = new SecureString("password".toCharArray()); - char[] hash = Hasher.BCRYPT.hash(password); + // Mocked users store is initiated with default hashing algorithm + final Hasher hasher = Hasher.resolve("bcrypt"); + char[] hash = hasher.hash(password); ReservedUserInfo userInfo = new ReservedUserInfo(hash, true, false); mockGetAllReservedUserInfo(usersStore, Collections.singletonMap("elastic", userInfo)); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); if (randomBoolean()) { PlainActionFuture future = new PlainActionFuture<>(); @@ -309,7 +322,7 @@ public void testBootstrapElasticPasswordWorksOnceSecurityIndexExists() throws Ex when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); doAnswer((i) -> { @@ -318,8 +331,8 @@ public void testBootstrapElasticPasswordWorksOnceSecurityIndexExists() throws Ex return null; }).when(usersStore).getReservedUserInfo(eq("elastic"), any(ActionListener.class)); reservedRealm.doAuthenticate(new UsernamePasswordToken(new ElasticUser(true).principal(), - mockSecureSettings.getString("bootstrap.password")), - listener); + mockSecureSettings.getString("bootstrap.password")), + listener); final AuthenticationResult result = listener.get(); assertThat(result.getStatus(), is(AuthenticationResult.Status.SUCCESS)); } @@ -331,18 +344,20 @@ public void testBootstrapElasticPasswordFailsOnceElasticUserExists() throws Exce when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); SecureString password = new SecureString("password".toCharArray()); + // Mocked users store is initiated with default hashing algorithm + final Hasher hasher = Hasher.resolve("bcrypt"); doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[1]; - char[] hash = Hasher.BCRYPT.hash(password); + char[] hash = hasher.hash(password); ReservedUserInfo userInfo = new ReservedUserInfo(hash, true, false); callback.onResponse(userInfo); return null; }).when(usersStore).getReservedUserInfo(eq("elastic"), any(ActionListener.class)); reservedRealm.doAuthenticate(new UsernamePasswordToken(new ElasticUser(true).principal(), - mockSecureSettings.getString("bootstrap.password")), listener); + mockSecureSettings.getString("bootstrap.password")), listener); assertFailedAuthentication(listener, "elastic"); // now try with the real password listener = new PlainActionFuture<>(); @@ -358,12 +373,12 @@ public void testBootstrapElasticPasswordWorksBeforeSecurityIndexExists() throws when(securityIndex.indexExists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); reservedRealm.doAuthenticate(new UsernamePasswordToken(new ElasticUser(true).principal(), - mockSecureSettings.getString("bootstrap.password")), - listener); + mockSecureSettings.getString("bootstrap.password")), + listener); final AuthenticationResult result = listener.get(); assertThat(result.getStatus(), is(AuthenticationResult.Status.SUCCESS)); } @@ -376,7 +391,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexExists when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME); @@ -398,7 +413,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexDoesNo when(securityIndex.indexExists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); + new AnonymousUser(Settings.EMPTY), securityIndex, threadPool); PlainActionFuture listener = new PlainActionFuture<>(); final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java index 7295e48d00394..8fad4c73a45ee 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java @@ -58,7 +58,8 @@ public class FileRealmTests extends ESTestCase { public void init() throws Exception { userPasswdStore = mock(FileUserPasswdStore.class); userRolesStore = mock(FileUserRolesStore.class); - globalSettings = Settings.builder().put("path.home", createTempDir()).build(); + globalSettings = Settings.builder().put("path.home", createTempDir()).put("xpack.security.authc.password_hashing.algorithm", + randomFrom("bcrypt9", "pbkdf2")).build(); threadPool = mock(ThreadPool.class); threadContext = new ThreadContext(globalSettings); when(threadPool.getThreadContext()).thenReturn(threadContext); @@ -85,10 +86,10 @@ public void testAuthenticate() throws Exception { public void testAuthenticateCaching() throws Exception { Settings settings = Settings.builder() - .put("cache.hash_algo", Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT)) - .build(); + .put("cache.hash_algo", Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT)).build(); RealmConfig config = new RealmConfig("file-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings), threadContext); + when(userPasswdStore.verifyPassword(eq("user1"), eq(new SecureString("test123")), any(Supplier.class))) .thenAnswer(VERIFY_PASSWORD_ANSWER); when(userRolesStore.roles("user1")).thenReturn(new String[]{"role1", "role2"}); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java index 367313c58a6cb..739952af63ee7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStoreTests.java @@ -53,9 +53,11 @@ public class FileUserPasswdStoreTests extends ESTestCase { @Before public void init() { settings = Settings.builder() - .put("resource.reload.interval.high", "2s") - .put("path.home", createTempDir()) - .build(); + .put("resource.reload.interval.high", "2s") + .put("path.home", createTempDir()) + .put("xpack.security.authc.password_hashing.algorithm", randomFrom("bcrypt", "bcrypt11", "pbkdf2", "pbkdf2_1000", + "pbkdf2_50000")) + .build(); env = TestEnvironment.newEnvironment(settings); threadPool = new TestThreadPool("test"); } @@ -86,17 +88,18 @@ public void testStore_AutoReload() throws Exception { Files.createDirectories(xpackConf); Path file = xpackConf.resolve("users"); Files.copy(users, file, StandardCopyOption.REPLACE_EXISTING); - + final Hasher hasher = Hasher.resolve(settings.get("xpack.security.authc.password_hashing.algorithm")); Settings fileSettings = randomBoolean() ? Settings.EMPTY : Settings.builder().put("files.users", file.toAbsolutePath()).build(); RealmConfig config = new RealmConfig("file-test", fileSettings, settings, env, threadPool.getThreadContext()); ResourceWatcherService watcherService = new ResourceWatcherService(settings, threadPool); final CountDownLatch latch = new CountDownLatch(1); FileUserPasswdStore store = new FileUserPasswdStore(config, watcherService, latch::countDown); - - User user = new User("bcrypt"); - assertThat(store.userExists("bcrypt"), is(true)); - AuthenticationResult result = store.verifyPassword("bcrypt", new SecureString("test123"), () -> user); + //Test users share the hashing algorithm name for convenience + String username = settings.get("xpack.security.authc.password_hashing.algorithm"); + User user = new User(username); + assertThat(store.userExists(username), is(true)); + AuthenticationResult result = store.verifyPassword(username, new SecureString("test123"), () -> user); assertThat(result.getStatus(), is(AuthenticationResult.Status.SUCCESS)); assertThat(result.getUser(), is(user)); @@ -104,7 +107,7 @@ public void testStore_AutoReload() throws Exception { try (BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8, StandardOpenOption.APPEND)) { writer.newLine(); - writer.append("foobar:").append(new String(Hasher.BCRYPT.hash(new SecureString("barfoo")))); + writer.append("foobar:").append(new String(hasher.hash(new SecureString("barfoo")))); } if (!latch.await(5, TimeUnit.SECONDS)) { @@ -133,9 +136,10 @@ public void testStore_AutoReload_WithParseFailures() throws Exception { final CountDownLatch latch = new CountDownLatch(1); FileUserPasswdStore store = new FileUserPasswdStore(config, watcherService, latch::countDown); - - User user = new User("bcrypt"); - final AuthenticationResult result = store.verifyPassword("bcrypt", new SecureString("test123"), () -> user); + //Test users share the hashing algorithm name for convenience + String username = settings.get("xpack.security.authc.password_hashing.algorithm"); + User user = new User(username); + final AuthenticationResult result = store.verifyPassword(username, new SecureString("test123"), () -> user); assertThat(result.getStatus(), is(AuthenticationResult.Status.SUCCESS)); assertThat(result.getUser(), is(user)); @@ -155,11 +159,11 @@ public void testParseFile() throws Exception { Path path = getDataPath("users"); Map users = FileUserPasswdStore.parseFile(path, null, Settings.EMPTY); assertThat(users, notNullValue()); - assertThat(users.size(), is(6)); + assertThat(users.size(), is(11)); assertThat(users.get("bcrypt"), notNullValue()); assertThat(new String(users.get("bcrypt")), equalTo("$2a$05$zxnP0vdREMxnEpkLCDI2OuSaSk/QEKA2.A42iOpI6U2u.RLLOWm1e")); assertThat(users.get("bcrypt10"), notNullValue()); - assertThat(new String(users.get("bcrypt10")), equalTo("$2y$10$FMhmFjwU5.qxQ/BsEciS9OqcJVkFMgXMo4uH5CelOR1j4N9zIv67e")); + assertThat(new String(users.get("bcrypt10")), equalTo("$2a$10$cFxpMx6YDrH/PXwLpTlux.KVykN1TG2Pgdl5oJX5/G/KYp3G6jbFG")); assertThat(users.get("md5"), notNullValue()); assertThat(new String(users.get("md5")), equalTo("$apr1$R3DdqiAZ$aljIkaIVPSarmDMlJUBBP.")); assertThat(users.get("crypt"), notNullValue()); @@ -168,6 +172,15 @@ public void testParseFile() throws Exception { assertThat(new String(users.get("plain")), equalTo("{plain}test123")); assertThat(users.get("sha"), notNullValue()); assertThat(new String(users.get("sha")), equalTo("{SHA}cojt0Pw//L6ToM8G41aOKFIWh7w=")); + assertThat(users.get("pbkdf2"), notNullValue()); + assertThat(new String(users.get("pbkdf2")), + equalTo("{PBKDF2}10000$ekcItXk4jtK2bBjbVk0rZuWRjT0DoQqQJOIfyMeLIxg=$RA2/Nn1jRi8QskRS5IVotCV0FBO6M8DlNXC37GKa/8c=")); + assertThat(users.get("pbkdf2_1000"), notNullValue()); + assertThat(new String(users.get("pbkdf2_1000")), + equalTo("{PBKDF2}1000$32yPZSShxuKYAl47ip0g6VwbFrD8tvFJuQCoRPGhXC8=$cXAE1BkBXRmkv7pQA7fw4TZ1+rFWS2/nZGeA3kL1Eu8=")); + assertThat(users.get("pbkdf2_50000"), notNullValue()); + assertThat(new String(users.get("pbkdf2_50000")), + equalTo("{PBKDF2}50000$z1CLJt0MEFjkIK5iEfgvfnA6xq7lF25uasspsTKSo5Q=$XxCVLbaKDimOdyWgLCLJiyoiWpA/XDMe/xtVgn1r5Sg=")); } public void testParseFile_Empty() throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java index 38a6344f98e54..052758d83718c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealmTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.Realm; import org.elasticsearch.xpack.core.security.authc.RealmConfig; -import org.elasticsearch.xpack.core.security.authc.support.BCrypt; import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; @@ -29,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -61,12 +61,11 @@ public void stop() throws InterruptedException { } public void testSettings() throws Exception { - String hashAlgo = randomFrom("bcrypt", "bcrypt4", "bcrypt5", "bcrypt6", "bcrypt7", "bcrypt8", "bcrypt9", - "sha1", "ssha256", "md5", "clear_text", "noop"); + String cachingHashAlgo = Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT); int maxUsers = randomIntBetween(10, 100); TimeValue ttl = TimeValue.timeValueMinutes(randomIntBetween(10, 20)); Settings settings = Settings.builder() - .put(CachingUsernamePasswordRealmSettings.CACHE_HASH_ALGO_SETTING.getKey(), hashAlgo) + .put(CachingUsernamePasswordRealmSettings.CACHE_HASH_ALGO_SETTING.getKey(), cachingHashAlgo) .put(CachingUsernamePasswordRealmSettings.CACHE_MAX_USERS_SETTING.getKey(), maxUsers) .put(CachingUsernamePasswordRealmSettings.CACHE_TTL_SETTING.getKey(), ttl) .build(); @@ -84,8 +83,7 @@ protected void doLookupUser(String username, ActionListener listener) { listener.onFailure(new UnsupportedOperationException("this method should not be called")); } }; - - assertThat(realm.hasher, sameInstance(Hasher.resolve(hashAlgo))); + assertThat(realm.cacheHasher, sameInstance(Hasher.resolve(cachingHashAlgo))); } public void testAuthCache() { @@ -347,8 +345,8 @@ public void testSingleAuthPerUserLimit() throws Exception { final String username = "username"; final SecureString password = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; final AtomicInteger authCounter = new AtomicInteger(0); - - final String passwordHash = new String(Hasher.BCRYPT.hash(password)); + final Hasher pwdHasher = Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")); + final String passwordHash = new String(pwdHasher.hash(password)); RealmConfig config = new RealmConfig("test_realm", Settings.EMPTY, globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY)); final CachingUsernamePasswordRealm realm = new CachingUsernamePasswordRealm("test", config, threadPool) { @@ -356,7 +354,7 @@ public void testSingleAuthPerUserLimit() throws Exception { protected void doAuthenticate(UsernamePasswordToken token, ActionListener listener) { authCounter.incrementAndGet(); // do something slow - if (BCrypt.checkpw(token.credentials(), passwordHash)) { + if (pwdHasher.verify(token.credentials(), passwordHash.toCharArray())) { listener.onResponse(AuthenticationResult.success(new User(username, new String[]{"r1", "r2", "r3"}))); } else { listener.onFailure(new IllegalStateException("password auth should never fail")); @@ -413,15 +411,15 @@ public void testCacheConcurrency() throws Exception { final String username = "username"; final SecureString password = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; final SecureString randomPassword = new SecureString(randomAlphaOfLength(password.length()).toCharArray()); - - final String passwordHash = new String(Hasher.BCRYPT.hash(password)); + final Hasher localHasher = Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9")); + final String passwordHash = new String(localHasher.hash(password)); RealmConfig config = new RealmConfig("test_realm", Settings.EMPTY, globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(Settings.EMPTY)); final CachingUsernamePasswordRealm realm = new CachingUsernamePasswordRealm("test", config, threadPool) { @Override protected void doAuthenticate(UsernamePasswordToken token, ActionListener listener) { // do something slow - if (BCrypt.checkpw(token.credentials(), passwordHash)) { + if (localHasher.verify(token.credentials(), passwordHash.toCharArray())) { listener.onResponse(AuthenticationResult.success(new User(username, new String[]{"r1", "r2", "r3"}))); } else { listener.onResponse(AuthenticationResult.unsuccessful("Incorrect password", null)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java index 0a8e8e9ac3936..c303c0ab4683a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.sameInstance; public class HasherTests extends ESTestCase { @@ -20,6 +21,21 @@ public void testBcryptFamilySelfGenerated() throws Exception { testHasherSelfGenerated(Hasher.BCRYPT7); testHasherSelfGenerated(Hasher.BCRYPT8); testHasherSelfGenerated(Hasher.BCRYPT9); + testHasherSelfGenerated(Hasher.BCRYPT10); + testHasherSelfGenerated(Hasher.BCRYPT11); + testHasherSelfGenerated(Hasher.BCRYPT12); + testHasherSelfGenerated(Hasher.BCRYPT13); + testHasherSelfGenerated(Hasher.BCRYPT14); + } + + public void testPBKDF2FamilySelfGenerated() throws Exception { + testHasherSelfGenerated(Hasher.PBKDF2); + testHasherSelfGenerated(Hasher.PBKDF2_1000); + testHasherSelfGenerated(Hasher.PBKDF2_10000); + testHasherSelfGenerated(Hasher.PBKDF2_50000); + testHasherSelfGenerated(Hasher.PBKDF2_100000); + testHasherSelfGenerated(Hasher.PBKDF2_500000); + testHasherSelfGenerated(Hasher.PBKDF2_1000000); } public void testMd5SelfGenerated() throws Exception { @@ -38,7 +54,7 @@ public void testNoopSelfGenerated() throws Exception { testHasherSelfGenerated(Hasher.NOOP); } - public void testResolve() throws Exception { + public void testResolve() { assertThat(Hasher.resolve("bcrypt"), sameInstance(Hasher.BCRYPT)); assertThat(Hasher.resolve("bcrypt4"), sameInstance(Hasher.BCRYPT4)); assertThat(Hasher.resolve("bcrypt5"), sameInstance(Hasher.BCRYPT5)); @@ -46,23 +62,80 @@ public void testResolve() throws Exception { assertThat(Hasher.resolve("bcrypt7"), sameInstance(Hasher.BCRYPT7)); assertThat(Hasher.resolve("bcrypt8"), sameInstance(Hasher.BCRYPT8)); assertThat(Hasher.resolve("bcrypt9"), sameInstance(Hasher.BCRYPT9)); + assertThat(Hasher.resolve("bcrypt10"), sameInstance(Hasher.BCRYPT)); + assertThat(Hasher.resolve("bcrypt11"), sameInstance(Hasher.BCRYPT11)); + assertThat(Hasher.resolve("bcrypt12"), sameInstance(Hasher.BCRYPT12)); + assertThat(Hasher.resolve("bcrypt13"), sameInstance(Hasher.BCRYPT13)); + assertThat(Hasher.resolve("bcrypt14"), sameInstance(Hasher.BCRYPT14)); + assertThat(Hasher.resolve("pbkdf2"), sameInstance(Hasher.PBKDF2)); + assertThat(Hasher.resolve("pbkdf2_1000"), sameInstance(Hasher.PBKDF2_1000)); + assertThat(Hasher.resolve("pbkdf2_10000"), sameInstance(Hasher.PBKDF2)); + assertThat(Hasher.resolve("pbkdf2_50000"), sameInstance(Hasher.PBKDF2_50000)); + assertThat(Hasher.resolve("pbkdf2_100000"), sameInstance(Hasher.PBKDF2_100000)); + assertThat(Hasher.resolve("pbkdf2_500000"), sameInstance(Hasher.PBKDF2_500000)); + assertThat(Hasher.resolve("pbkdf2_1000000"), sameInstance(Hasher.PBKDF2_1000000)); assertThat(Hasher.resolve("sha1"), sameInstance(Hasher.SHA1)); assertThat(Hasher.resolve("md5"), sameInstance(Hasher.MD5)); assertThat(Hasher.resolve("ssha256"), sameInstance(Hasher.SSHA256)); assertThat(Hasher.resolve("noop"), sameInstance(Hasher.NOOP)); assertThat(Hasher.resolve("clear_text"), sameInstance(Hasher.NOOP)); - try { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { Hasher.resolve("unknown_hasher"); - fail("expected a settings error when trying to resolve an unknown hasher"); - } catch (IllegalArgumentException e) { - // expected - } - Hasher hasher = randomFrom(Hasher.values()); - assertThat(Hasher.resolve("unknown_hasher", hasher), sameInstance(hasher)); + }); + assertThat(e.getMessage(), containsString("unknown hash function ")); + } + + public void testResolveFromHash() { + assertThat(Hasher.resolveFromHash("$2a$10$1oZj.8KmlwiCy4DWKvDH3OU0Ko4WRF4FknyvCh3j/ZtaRCNYA6Xzm".toCharArray()), + sameInstance(Hasher.BCRYPT)); + assertThat(Hasher.resolveFromHash("$2a$04$GwJtIQiGMHASEYphMiCpjeZh1cDyYC5U.DKfNKa4i/y0IbOvc2LiG".toCharArray()), + sameInstance(Hasher.BCRYPT4)); + assertThat(Hasher.resolveFromHash("$2a$05$xLmwSB7Nw7PcqP.6hXdc4eUZbT.4.iAZ3CTPzSaUibrrYjC6Vwq1m".toCharArray()), + sameInstance(Hasher.BCRYPT5)); + assertThat(Hasher.resolveFromHash("$2a$06$WQX1MALAjVOhR2YKmLcHYed2oROzBl3OZPtvq3FkVZYwm9X2LVKYm".toCharArray()), + sameInstance(Hasher.BCRYPT6)); + assertThat(Hasher.resolveFromHash("$2a$07$Satxnu2fCvwYXpHIk8A2sO2uwROrsV7WrNiRJPq1oXEl5lc9FE.7S".toCharArray()), + sameInstance(Hasher.BCRYPT7)); + assertThat(Hasher.resolveFromHash("$2a$08$LLfkTt2C9TUl5sDtgqmE3uRw9nHt748d3eMSGfbFYgQQQhjbXHFo2".toCharArray()), + sameInstance(Hasher.BCRYPT8)); + assertThat(Hasher.resolveFromHash("$2a$09$.VCWA3yFVdd6gfI526TUrufb4TvxMuhW0jIuMfhd4/fy1Ak/zrSFe".toCharArray()), + sameInstance(Hasher.BCRYPT9)); + assertThat(Hasher.resolveFromHash("$2a$10$OEiXFrUUY02Nm7YsEgzFuuJ3yO3HAYzJUU7omseluy28s7FYaictu".toCharArray()), + sameInstance(Hasher.BCRYPT)); + assertThat(Hasher.resolveFromHash("$2a$11$Ya53LCozFlKABu05xsAbj.9xmrczyuAY/fTvxKkDiHOJc5GYcaNRy".toCharArray()), + sameInstance(Hasher.BCRYPT11)); + assertThat(Hasher.resolveFromHash("$2a$12$oUW2hiWBHYwbJamWi6YDPeKS2NBCvD4GR50zh9QZCcgssNFcbpg/a".toCharArray()), + sameInstance(Hasher.BCRYPT12)); + assertThat(Hasher.resolveFromHash("$2a$13$0PDx6mxKK4bLSgpc5H6eaeylWub7UFghjxV03lFYSz4WS4slDT30q".toCharArray()), + sameInstance(Hasher.BCRYPT13)); + assertThat(Hasher.resolveFromHash("$2a$14$lFyXmX7p9/FHr7W4nxTnfuCkjAoBHv6awQlv8jlKZ/YCMI65i38e6".toCharArray()), + sameInstance(Hasher.BCRYPT14)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}1000$oNl3JWiDZhXqhrpk9Kl+T0tKpVNNV3UHNxENPePpo2M=$g9lERDX5op20eX534bHdQy7ySRwobxwtaxxsz3AYPIU=".toCharArray()), + sameInstance(Hasher.PBKDF2_1000)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}10000$UrwrHBY4GA1na9KxRpoFkUiICTeZe+mMZCZOg6bRSLc=$1Wl32wRQ9Q3Sv1IFoNwgSrUa5YifLv0MoxAO6leyip8=".toCharArray()), + sameInstance(Hasher.PBKDF2)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}50000$mxa5m9AlgtKLUXKi/pE5+4w7ZexGSOtlUHD043NHVdc=$LE5Ncph672M8PtugfRgk2k3ue9qY2cKgiguuAd+e3I0=".toCharArray()), + sameInstance(Hasher.PBKDF2_50000)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}100000$qFs8H0FjietnI7sgr/1Av4H+Z7d/9dehfZ2ptU474jk=$OFj40Ha0XcHWUXSspRx6EeXnTcuN0Nva2/i2c/hvnZE=".toCharArray()), + sameInstance(Hasher.PBKDF2_100000)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}500000$wyttuDlppd5KYD35uDZN6vudB50Cjshm5efZhOxZZQI=$ADZpOHY6llJZsjupZCn6s4Eocg0dKKdBiNjDBYqhlzA=".toCharArray()), + sameInstance(Hasher.PBKDF2_500000)); + assertThat(Hasher.resolveFromHash( + "{PBKDF2}1000000$UuyhtjDEzWmE2wyY80akZKPWWpy2r2X50so41YML82U=$WFasYLelqbjQwt3EqFlUcwHiC38EZC45Iu/Iz0xL1GQ=".toCharArray()), + sameInstance(Hasher.PBKDF2_1000000)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + Hasher.resolveFromHash("{GBGN}cGR8S2vr3FuFuOpQitR".toCharArray()); + }); + assertThat(e.getMessage(), containsString("unknown hash format for hash")); } - private static void testHasherSelfGenerated(Hasher hasher) throws Exception { - SecureString passwd = new SecureString(randomAlphaOfLength(10)); + private static void testHasherSelfGenerated(Hasher hasher) { + SecureString passwd = new SecureString(randomAlphaOfLength(10).toCharArray()); char[] hash = hasher.hash(passwd); assertTrue(hasher.verify(passwd, hash)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java index b8634f2e9f358..3c3574704ac9c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import java.util.Collections; @@ -17,13 +16,14 @@ import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; public class AnalyzeTests extends SecurityIntegTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("test123".toCharArray()))); @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(new SecureString + ("test123".toCharArray()))); return super.configUsers() + - "analyze_indices:" + USERS_PASSWD_HASHED + "\n" + - "analyze_cluster:" + USERS_PASSWD_HASHED + "\n"; + "analyze_indices:" + usersPasswdHashed + "\n" + + "analyze_cluster:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java index d273d61959e2a..c6cb8bb662c7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.SecurityIntegTestCase; -import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.junit.Before; import java.util.Collections; @@ -31,16 +30,16 @@ public class IndexAliasesTests extends SecurityIntegTestCase { - protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecureString("test123".toCharArray()))); - @Override protected String configUsers() { + final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(new SecureString + ("test123".toCharArray()))); return super.configUsers() + - "create_only:" + USERS_PASSWD_HASHED + "\n" + - "create_test_aliases_test:" + USERS_PASSWD_HASHED + "\n" + - "create_test_aliases_alias:" + USERS_PASSWD_HASHED + "\n" + - "create_test_aliases_test_alias:" + USERS_PASSWD_HASHED + "\n" + - "aliases_only:" + USERS_PASSWD_HASHED + "\n"; + "create_only:" + usersPasswdHashed + "\n" + + "create_test_aliases_test:" + usersPasswdHashed + "\n" + + "create_test_aliases_alias:" + usersPasswdHashed + "\n" + + "create_test_aliases_test_alias:" + usersPasswdHashed + "\n" + + "aliases_only:" + usersPasswdHashed + "\n"; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java index c3d24e1adc7a4..4466190f83ded 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java @@ -32,7 +32,9 @@ public void testScrollIsPerUser() throws Exception { securityClient().preparePutRole("scrollable") .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null) .get(); - securityClient().preparePutUser("other", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), "scrollable").get(); + securityClient().preparePutUser("other", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), getFastStoredHashAlgoForTests(), + "scrollable") + .get(); final int numDocs = randomIntBetween(4, 16); IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java index 677be9a94e7ce..55cd659509bfe 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java @@ -48,7 +48,8 @@ protected void doRun() throws Exception { for (int i = 0; i < numRequests; i++) { requests.add(securityClient() .preparePutUser("user" + userNumber.getAndIncrement(), "password".toCharArray(), - randomAlphaOfLengthBetween(1, 16)) + getFastStoredHashAlgoForTests(), + randomAlphaOfLengthBetween(1, 16)) .request()); } diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/file/users b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/file/users index 997c839c2695b..435c3fc5ed9e2 100644 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/file/users +++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/file/users @@ -5,4 +5,9 @@ plain:{plain}test123 sha:{SHA}cojt0Pw//L6ToM8G41aOKFIWh7w= # this is a comment line # another comment line -bcrypt10:$2y$10$FMhmFjwU5.qxQ/BsEciS9OqcJVkFMgXMo4uH5CelOR1j4N9zIv67e \ No newline at end of file +pbkdf2:{PBKDF2}10000$ekcItXk4jtK2bBjbVk0rZuWRjT0DoQqQJOIfyMeLIxg=$RA2/Nn1jRi8QskRS5IVotCV0FBO6M8DlNXC37GKa/8c= +pbkdf2_1000:{PBKDF2}1000$32yPZSShxuKYAl47ip0g6VwbFrD8tvFJuQCoRPGhXC8=$cXAE1BkBXRmkv7pQA7fw4TZ1+rFWS2/nZGeA3kL1Eu8= +pbkdf2_50000:{PBKDF2}50000$z1CLJt0MEFjkIK5iEfgvfnA6xq7lF25uasspsTKSo5Q=$XxCVLbaKDimOdyWgLCLJiyoiWpA/XDMe/xtVgn1r5Sg= +bcrypt9:$2a$09$YhstxoAjO7M5MtAIFv7dVO70/pElJAYrzyumeCpLpZV2Gcz4J2/F. +bcrypt10:$2a$10$cFxpMx6YDrH/PXwLpTlux.KVykN1TG2Pgdl5oJX5/G/KYp3G6jbFG +bcrypt11:$2a$11$uxr7b0qgCrLV9VIz9XS7M.Eoc0gJRR60oV48UK5DKfLOp.9HjfYF2 \ No newline at end of file diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java index 4e1fb72256086..bd3c53a3b41be 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; @@ -52,7 +53,7 @@ protected Collection> transportClientPlugins() { public void setupTestUser(String role) { SecurityClient securityClient = new SecurityClient(client()); - securityClient.preparePutUser(TEST_USER, TEST_PWD.toCharArray(), role).get(); + securityClient.preparePutUser(TEST_USER, TEST_PWD.toCharArray(), Hasher.BCRYPT, role).get(); } public void testAuthorizedCustomRoleSucceeds() throws Exception { diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java index 719971bf8a829..4ac927c6646c1 100644 --- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java +++ b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse; import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse; import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; @@ -46,7 +47,7 @@ public void setupUpTest() throws Exception { SecurityClient c = new SecurityClient(client); // Add an existing user so the tool will skip it - PutUserResponse pur = c.preparePutUser("existing", "s3kirt".toCharArray(), "role1", "user").get(); + PutUserResponse pur = c.preparePutUser("existing", "s3kirt".toCharArray(), Hasher.BCRYPT, "role1", "user").get(); assertTrue(pur.created()); } diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java index f8cbfaea4955d..d80f98964f3a6 100644 --- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java +++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java @@ -54,6 +54,8 @@ public class UsersToolTests extends CommandTestCase { // settings used to create an Environment for tools Settings settings; + Hasher hasher; + @BeforeClass public static void setupJimfs() throws IOException { String view = randomFrom("basic", "posix"); @@ -68,11 +70,12 @@ public void setupHome() throws IOException { IOUtils.rm(homeDir); confDir = homeDir.resolve("config"); Files.createDirectories(confDir); + hasher = Hasher.resolve(randomFrom("bcrypt", "pbkdf2")); String defaultPassword = SecuritySettingsSourceField.TEST_PASSWORD; Files.write(confDir.resolve("users"), Arrays.asList( - "existing_user:" + new String(Hasher.BCRYPT.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)), - "existing_user2:" + new String(Hasher.BCRYPT.hash(new SecureString((defaultPassword + "2").toCharArray()))), - "existing_user3:" + new String(Hasher.BCRYPT.hash(new SecureString((defaultPassword + "3").toCharArray()))) + "existing_user:" + new String(hasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)), + "existing_user2:" + new String(hasher.hash(new SecureString((defaultPassword + "2").toCharArray()))), + "existing_user3:" + new String(hasher.hash(new SecureString((defaultPassword + "3").toCharArray()))) ), StandardCharsets.UTF_8); Files.write(confDir.resolve("users_roles"), Arrays.asList( "test_admin:existing_user,existing_user2", @@ -170,9 +173,10 @@ void assertUser(String username, String password) throws IOException { continue; } String gotHash = usernameHash[1]; - SecureString expectedHash = new SecureString(password); - assertTrue("Expected hash " + expectedHash + " for password " + password + " but got " + gotHash, - Hasher.BCRYPT.verify(expectedHash, gotHash.toCharArray())); + SecureString expectedHash = new SecureString(password.toCharArray()); + // CommandTestCase#execute runs passwd with default settings, so bcrypt with cost of 10 + Hasher bcryptHasher = Hasher.resolve("bcrypt"); + assertTrue("Could not validate password for user", bcryptHasher.verify(expectedHash, gotHash.toCharArray())); return; } fail("Could not find username " + username + " in users file:\n" + lines.toString()); From 02c01cb4cf1f9fb4d5d6067ce0a0654df91d2af7 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 28 Jun 2018 12:12:55 -0700 Subject: [PATCH 05/18] Fix CreateSnapshotRequestTests Failure (#31630) Original test failure found here in issue (#31625). Had to rework the tests to only include options available externally for create snapshot requests. --- .../action/support/IndicesOptions.java | 3 + .../create/CreateSnapshotRequestTests.java | 37 +++++---- .../action/support/IndicesOptionsTests.java | 82 +++++++++++++++++++ 3 files changed, 106 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 93641574bde12..19572a6c212a2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -325,6 +325,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); builder.field("ignore_unavailable", ignoreUnavailable()); builder.field("allow_no_indices", allowNoIndices()); + builder.field("forbid_aliases_to_multiple_indices", allowAliasesToMultipleIndices() == false); + builder.field("forbid_closed_indices", forbidClosedIndices()); + builder.field("ignore_aliases", ignoreAliases()); return builder; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 9e484217870eb..1bde8ab572b72 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -20,8 +20,11 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndicesOptions.Option; +import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent.MapParams; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -30,6 +33,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,14 +44,13 @@ public class CreateSnapshotRequestTests extends ESTestCase { // tests creating XContent and parsing with source(Map) equivalency - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31625") public void testToXContent() throws IOException { String repo = randomAlphaOfLength(5); String snap = randomAlphaOfLength(10); CreateSnapshotRequest original = new CreateSnapshotRequest(repo, snap); - if (randomBoolean()) { // replace + if (randomBoolean()) { List indices = new ArrayList<>(); int count = randomInt(3) + 1; @@ -55,11 +61,11 @@ public void testToXContent() throws IOException { original.indices(indices); } - if (randomBoolean()) { // replace + if (randomBoolean()) { original.partial(randomBoolean()); } - if (randomBoolean()) { // replace + if (randomBoolean()) { Map settings = new HashMap<>(); int count = randomInt(3) + 1; @@ -67,32 +73,31 @@ public void testToXContent() throws IOException { settings.put(randomAlphaOfLength(randomInt(3) + 2), randomAlphaOfLength(randomInt(3) + 2)); } + original.settings(settings); } - if (randomBoolean()) { // replace + if (randomBoolean()) { original.includeGlobalState(randomBoolean()); } - if (randomBoolean()) { // replace - IndicesOptions[] indicesOptions = new IndicesOptions[] { - IndicesOptions.STRICT_EXPAND_OPEN, - IndicesOptions.STRICT_EXPAND_OPEN_CLOSED, - IndicesOptions.LENIENT_EXPAND_OPEN, - IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED, - IndicesOptions.STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED}; + if (randomBoolean()) { + Collection wildcardStates = randomSubsetOf(Arrays.asList(WildcardStates.values())); + Collection

- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config} - * in order to run these tests. - */ -@ThirdParty -public abstract class AbstractAwsTestCase extends ESIntegTestCase { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder settings = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put("cloud.aws.test.random", randomInt()) - .put("cloud.aws.test.write_failures", 0.1) - .put("cloud.aws.test.read_failures", 0.1); - - // if explicit, just load it and don't load from env - try { - if (Strings.hasText(System.getProperty("tests.config"))) { - try { - settings.loadFromPath(PathUtils.get(System.getProperty("tests.config"))); - } catch (IOException e) { - throw new IllegalArgumentException("could not load aws tests config", e); - } - } else { - throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml"); - } - } catch (SettingsException exception) { - throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception); - } - return settings.build(); - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(TestAwsS3Service.TestPlugin.class); - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java deleted file mode 100644 index 439927acb4e70..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; -import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; -import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.repositories.RepositoryMissingException; -import org.elasticsearch.repositories.RepositoryVerificationException; -import org.elasticsearch.snapshots.SnapshotMissingException; -import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.After; -import org.junit.Before; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; - -@ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) -public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { - - private String basePath; - - @Before - public final void wipeBefore() { - wipeRepositories(); - basePath = "repo-" + randomInt(); - cleanRepositoryFiles(basePath); - } - - @After - public final void wipeAfter() { - wipeRepositories(); - cleanRepositoryFiles(basePath); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") - public void testEncryption() { - Client client = client(); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); - - Settings repositorySettings = Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000)) - .put(S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true) - .build(); - - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(repositorySettings).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - createIndex("test-idx-1", "test-idx-2", "test-idx-3"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); - index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); - index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - - Settings settings = internalCluster().getInstance(Settings.class); - Settings bucket = settings.getByPrefix("repositories.s3."); - try (AmazonS3Reference s3Client = internalCluster().getInstance(S3Service.class).client("default")) { - String bucketName = bucket.get("bucket"); - logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); - List summaries = s3Client.client().listObjects(bucketName, basePath).getObjectSummaries(); - for (S3ObjectSummary summary : summaries) { - assertThat(s3Client.client().getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256")); - } - } - - logger.info("--> delete some data"); - for (int i = 0; i < 50; i++) { - client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); - } - for (int i = 50; i < 100; i++) { - client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); - } - for (int i = 0; i < 100; i += 2) { - client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(50L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(50L)); - assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L)); - - logger.info("--> close indices"); - client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); - - logger.info("--> restore all indices from the snapshot"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); - assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - ensureGreen(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L)); - - // Test restore after index deletion - logger.info("--> delete indices"); - cluster().wipeIndices("test-idx-1", "test-idx-2"); - logger.info("--> restore one index after deletion"); - restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); - assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - ensureGreen(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); - assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); - assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); - } - - /** - * This test verifies that the test configuration is set up in a manner that - * does not make the test {@link #testRepositoryWithCustomCredentials()} pointless. - */ - public void testRepositoryWithCustomCredentialsIsNotAccessibleByDefaultCredentials() { - Client client = client(); - Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - try { - client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - .put(S3Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket")) - ).get(); - fail("repository verification should have raise an exception!"); - } catch (RepositoryVerificationException e) { - } - } - - public void testRepositoryWithBasePath() { - Client client = client(); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - assertRepositoryIsOperational(client, "test-repo"); - } - - public void testRepositoryWithCustomCredentials() { - Client client = client(); - Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - .put(S3Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket")) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - assertRepositoryIsOperational(client, "test-repo"); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") - public void testRepositoryWithCustomEndpointProtocol() { - Client client = client(); - Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.external-bucket."); - logger.info("--> creating s3 repostoriy with endpoint [{}], bucket[{}] and path [{}]", bucketSettings.get("endpoint"), bucketSettings.get("bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket")) - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - assertRepositoryIsOperational(client, "test-repo"); - } - - /** - * This test verifies that the test configuration is set up in a manner that - * does not make the test {@link #testRepositoryInRemoteRegion()} pointless. - */ - public void testRepositoryInRemoteRegionIsRemote() { - Client client = client(); - Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - try { - client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - .put(S3Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket")) - // Below setting intentionally omitted to assert bucket is not available in default region. - // .put("region", privateBucketSettings.get("region")) - ).get(); - fail("repository verification should have raise an exception!"); - } catch (RepositoryVerificationException e) { - } - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") - public void testRepositoryInRemoteRegion() { - Client client = client(); - Settings settings = internalCluster().getInstance(Settings.class); - Settings bucketSettings = settings.getByPrefix("repositories.s3.remote-bucket."); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - .put(S3Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket")) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - assertRepositoryIsOperational(client, "test-repo"); - } - - /** - * Test case for issue #86: https://github.com/elastic/elasticsearch-cloud-aws/issues/86 - */ - public void testNonExistingRepo86() { - Client client = client(); - logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); - PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - logger.info("--> restore non existing snapshot"); - try { - client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).execute().actionGet(); - fail("Shouldn't be here"); - } catch (SnapshotMissingException ex) { - // Expected - } - } - - /** - * For issue #86: https://github.com/elastic/elasticsearch-cloud-aws/issues/86 - */ - public void testGetDeleteNonExistingSnapshot86() { - ClusterAdminClient client = client().admin().cluster(); - logger.info("--> creating s3 repository without any path"); - PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo") - .setType("s3").setSettings(Settings.builder() - .put(S3Repository.BASE_PATH_SETTING.getKey(), basePath) - ).get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - try { - client.prepareGetSnapshots("test-repo").addSnapshots("no-existing-snapshot").get(); - fail("Shouldn't be here"); - } catch (SnapshotMissingException ex) { - // Expected - } - - try { - client.prepareDeleteSnapshot("test-repo", "no-existing-snapshot").get(); - fail("Shouldn't be here"); - } catch (SnapshotMissingException ex) { - // Expected - } - } - - private void assertRepositoryIsOperational(Client client, String repository) { - createIndex("test-idx-1"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repository, "test-snap").setWaitForCompletion(true).setIndices("test-idx-*").get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertThat(client.admin().cluster().prepareGetSnapshots(repository).setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - - logger.info("--> delete some data"); - for (int i = 0; i < 50; i++) { - client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(50L)); - - logger.info("--> close indices"); - client.admin().indices().prepareClose("test-idx-1").get(); - - logger.info("--> restore all indices from the snapshot"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repository, "test-snap").setWaitForCompletion(true).execute().actionGet(); - assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - ensureGreen(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L)); - } - - - /** - * Deletes repositories, supports wildcard notation. - */ - public static void wipeRepositories(String... repositories) { - // if nothing is provided, delete all - if (repositories.length == 0) { - repositories = new String[]{"*"}; - } - for (String repository : repositories) { - try { - client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet(); - } catch (RepositoryMissingException ex) { - // ignore - } - } - } - - /** - * Deletes content of the repository files in the bucket - */ - public void cleanRepositoryFiles(String basePath) { - Settings settings = internalCluster().getInstance(Settings.class); - Settings[] buckets = { - settings.getByPrefix("repositories.s3."), - settings.getByPrefix("repositories.s3.private-bucket."), - settings.getByPrefix("repositories.s3.remote-bucket."), - settings.getByPrefix("repositories.s3.external-bucket.") - }; - for (Settings bucket : buckets) { - String bucketName = bucket.get("bucket"); - - // We check that settings has been set in elasticsearch.yml integration test file - // as described in README - assertThat("Your settings in elasticsearch.yml are incorrect. Check README file.", bucketName, notNullValue()); - try (AmazonS3Reference s3Client = internalCluster().getInstance(S3Service.class).client("default")) { - ObjectListing prevListing = null; - //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html - //we can do at most 1K objects per delete - //We don't know the bucket name until first object listing - DeleteObjectsRequest multiObjectDeleteRequest = null; - ArrayList keys = new ArrayList<>(); - while (true) { - ObjectListing list; - if (prevListing != null) { - list = s3Client.client().listNextBatchOfObjects(prevListing); - } else { - list = s3Client.client().listObjects(bucketName, basePath); - multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); - } - for (S3ObjectSummary summary : list.getObjectSummaries()) { - keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); - //Every 500 objects batch the delete request - if (keys.size() > 500) { - multiObjectDeleteRequest.setKeys(keys); - s3Client.client().deleteObjects(multiObjectDeleteRequest); - multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); - keys.clear(); - } - } - if (list.isTruncated()) { - prevListing = list; - } else { - break; - } - } - if (!keys.isEmpty()) { - multiObjectDeleteRequest.setKeys(keys); - s3Client.client().deleteObjects(multiObjectDeleteRequest); - } - } catch (Exception ex) { - logger.warn((Supplier) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex); - } - } - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTests.java deleted file mode 100644 index 667a75656b30e..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ProxiedSnapshotRestoreOverHttpsTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import org.elasticsearch.common.settings.Settings; -import org.junit.Before; - -/** - * This will only run if you define in your `elasticsearch.yml` file a s3 specific proxy - * cloud.aws.s3.proxy_host: mys3proxy.company.com - * cloud.aws.s3.proxy_port: 8080 - */ -public class S3ProxiedSnapshotRestoreOverHttpsTests extends AbstractS3SnapshotRestoreTest { - - private boolean proxySet = false; - - @Override - public Settings nodeSettings(int nodeOrdinal) { - Settings settings = super.nodeSettings(nodeOrdinal); - String proxyHost = settings.get("cloud.aws.s3.proxy_host"); - proxySet = proxyHost != null; - return settings; - } - - @Before - public void checkProxySettings() { - assumeTrue("we are expecting proxy settings in elasticsearch.yml file", proxySet); - } - -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTests.java deleted file mode 100644 index 1c1a3457d7a04..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import org.elasticsearch.common.settings.Settings; - -public class S3SnapshotRestoreOverHttpTests extends AbstractS3SnapshotRestoreTest { - @Override - public Settings nodeSettings(int nodeOrdinal) { - Settings.Builder settings = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("cloud.aws.s3.protocol", "http"); - return settings.build(); - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTests.java deleted file mode 100644 index b888d015836cd..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3SnapshotRestoreOverHttpsTests.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import org.elasticsearch.common.settings.Settings; - -public class S3SnapshotRestoreOverHttpsTests extends AbstractS3SnapshotRestoreTest { - @Override - public Settings nodeSettings(int nodeOrdinal) { - Settings.Builder settings = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put("cloud.aws.s3.protocol", "https"); - return settings.build(); - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java deleted file mode 100644 index 0c762659a5fe0..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAmazonS3.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.UploadPartRequest; -import com.amazonaws.services.s3.model.UploadPartResult; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; - -import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; - -public class TestAmazonS3 extends AmazonS3Wrapper { - - protected final Logger logger = Loggers.getLogger(getClass()); - - private double writeFailureRate = 0.0; - private double readFailureRate = 0.0; - - private final String randomPrefix; - - ConcurrentMap accessCounts = new ConcurrentHashMap<>(); - - private long incrementAndGet(String path) { - AtomicLong value = accessCounts.get(path); - if (value == null) { - value = accessCounts.putIfAbsent(path, new AtomicLong(1)); - } - if (value != null) { - return value.incrementAndGet(); - } - return 1; - } - - public TestAmazonS3(AmazonS3 delegate, Settings settings) { - super(delegate); - randomPrefix = settings.get("cloud.aws.test.random"); - writeFailureRate = settings.getAsDouble("cloud.aws.test.write_failures", 0.0); - readFailureRate = settings.getAsDouble("cloud.aws.test.read_failures", 0.0); - } - - @Override - public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) throws AmazonClientException, AmazonServiceException { - if (shouldFail(bucketName, key, writeFailureRate)) { - final long length = metadata.getContentLength(); - final long partToRead = (long) (length * randomDouble()); - final byte[] buffer = new byte[1024]; - for (long cur = 0; cur < partToRead; cur += buffer.length) { - try { - input.read(buffer, 0, (int) ((partToRead - cur) > buffer.length ? buffer.length : partToRead - cur)); - } catch (final IOException ex) { - throw new ElasticsearchException("cannot read input stream", ex); - } - } - logger.info("--> random write failure on putObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); - final AmazonS3Exception ex = new AmazonS3Exception("Random S3 exception"); - ex.setStatusCode(400); - ex.setErrorCode("RequestTimeout"); - throw ex; - } else { - return super.putObject(bucketName, key, input, metadata); - } - } - - @Override - public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException { - if (shouldFail(request.getBucketName(), request.getKey(), writeFailureRate)) { - final long length = request.getPartSize(); - final long partToRead = (long) (length * randomDouble()); - final byte[] buffer = new byte[1024]; - for (long cur = 0; cur < partToRead; cur += buffer.length) { - try (InputStream input = request.getInputStream()){ - input.read(buffer, 0, (int) ((partToRead - cur) > buffer.length ? buffer.length : partToRead - cur)); - } catch (final IOException ex) { - throw new ElasticsearchException("cannot read input stream", ex); - } - } - logger.info("--> random write failure on uploadPart method: throwing an exception for [bucket={}, key={}]", request.getBucketName(), request.getKey()); - final AmazonS3Exception ex = new AmazonS3Exception("Random S3 write exception"); - ex.setStatusCode(400); - ex.setErrorCode("RequestTimeout"); - throw ex; - } else { - return super.uploadPart(request); - } - } - - @Override - public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException { - if (shouldFail(bucketName, key, readFailureRate)) { - logger.info("--> random read failure on getObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); - final AmazonS3Exception ex = new AmazonS3Exception("Random S3 read exception"); - ex.setStatusCode(404); - throw ex; - } else { - return super.getObject(bucketName, key); - } - } - - private boolean shouldFail(String bucketName, String key, double probability) { - if (probability > 0.0) { - String path = randomPrefix + "-" + bucketName + "+" + key; - path += "/" + incrementAndGet(path); - return Math.abs(hashCode(path)) < (Integer.MAX_VALUE * probability); - } else { - return false; - } - } - - private int hashCode(String path) { - try { - final MessageDigest digest = MessageDigest.getInstance("MD5"); - final byte[] bytes = digest.digest(path.getBytes("UTF-8")); - int i = 0; - return ((bytes[i++] & 0xFF) << 24) | ((bytes[i++] & 0xFF) << 16) - | ((bytes[i++] & 0xFF) << 8) | (bytes[i++] & 0xFF); - } catch (final UnsupportedEncodingException ex) { - throw new ElasticsearchException("cannot calculate hashcode", ex); - } catch (final NoSuchAlgorithmException ex) { - throw new ElasticsearchException("cannot calculate hashcode", ex); - } - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java deleted file mode 100644 index 828d8ef850462..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/TestAwsS3Service.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import java.util.IdentityHashMap; - -import com.amazonaws.services.s3.AmazonS3; -import org.elasticsearch.common.settings.Settings; - -public class TestAwsS3Service extends S3Service { - public static class TestPlugin extends S3RepositoryPlugin { - public TestPlugin(Settings settings) { - super(settings, new TestAwsS3Service(settings)); - } - } - - IdentityHashMap clients = new IdentityHashMap<>(); - - public TestAwsS3Service(Settings settings) { - super(settings); - } - - @Override - public synchronized AmazonS3Reference client(String clientName) { - return new AmazonS3Reference(cachedWrapper(super.client(clientName))); - } - - private AmazonS3 cachedWrapper(AmazonS3Reference clientReference) { - TestAmazonS3 wrapper = clients.get(clientReference); - if (wrapper == null) { - wrapper = new TestAmazonS3(clientReference.client(), settings); - clients.put(clientReference, wrapper); - } - return wrapper; - } - - @Override - protected synchronized void releaseCachedClients() { - super.releaseCachedClients(); - clients.clear(); - } - -} diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index c79f432278ae8..bf9c81932348d 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -28,6 +28,8 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.snapshots.SnapshotMissingException; +import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; @@ -50,7 +52,7 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase protected abstract void createTestRepository(String name); public void testSnapshotAndRestore() throws Exception { - String repoName = randomAsciiName(); + final String repoName = randomAsciiName(); logger.info("--> creating repository {}", repoName); createTestRepository(repoName); int indexCount = randomIntBetween(1, 5); @@ -63,7 +65,7 @@ public void testSnapshotAndRestore() throws Exception { assertHitCount(client().prepareSearch(indexNames[i]).setSize(0).get(), docCounts[i]); } - String snapshotName = randomAsciiName(); + final String snapshotName = randomAsciiName(); logger.info("--> create snapshot {}:{}", repoName, snapshotName); assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .setWaitForCompletion(true).setIndices(indexNames)); @@ -109,6 +111,15 @@ public void testSnapshotAndRestore() throws Exception { logger.info("--> delete snapshot {}:{}", repoName, snapshotName); assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get()); + + expectThrows(SnapshotMissingException.class, () -> + client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(snapshotName).get()); + + expectThrows(SnapshotMissingException.class, () -> + client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get()); + + expectThrows(SnapshotRestoreException.class, () -> + client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName).setWaitForCompletion(randomBoolean()).get()); } public void testMultipleSnapshotAndRollback() throws Exception { @@ -166,7 +177,7 @@ public void testMultipleSnapshotAndRollback() throws Exception { } } - public void testIndicesDeletedFromRepository() throws Exception { + public void testIndicesDeletedFromRepository() { Client client = client(); logger.info("--> creating repository"); From 117e9066db785182335ac74c8bacad30858c222f Mon Sep 17 00:00:00 2001 From: Ioannis Kakavas Date: Fri, 29 Jun 2018 14:41:13 +0300 Subject: [PATCH 16/18] Support multiple system store types (#31650) Support multiple system store types When falling back to using the system keystore and - most usually - truststore, do not assume that it will be a JKS store, but deduct its type from {@code KeyStore#getDefaultKeyStoreType}. This allows the use of any store type the Security Provider supports by setting the keystore.type java security property. --- .../elasticsearch/xpack/core/ssl/SSLConfiguration.java | 9 +++++---- .../xpack/core/ssl/SSLConfigurationReloaderTests.java | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfiguration.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfiguration.java index 0f91abac2a73e..731d59a3ac078 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfiguration.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfiguration.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.file.Path; import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -194,9 +195,9 @@ private static KeyConfig createKeyConfig(Settings settings, SSLConfiguration glo if (System.getProperty("javax.net.ssl.keyStore") != null) { // TODO: we should not support loading a keystore from sysprops... try (SecureString keystorePassword = new SecureString(System.getProperty("javax.net.ssl.keyStorePassword", ""))) { - return new StoreKeyConfig(System.getProperty("javax.net.ssl.keyStore"), "jks", keystorePassword, keystorePassword, - System.getProperty("ssl.KeyManagerFactory.algorithm", KeyManagerFactory.getDefaultAlgorithm()), - System.getProperty("ssl.TrustManagerFactory.algorithm", TrustManagerFactory.getDefaultAlgorithm())); + return new StoreKeyConfig(System.getProperty("javax.net.ssl.keyStore"), KeyStore.getDefaultType(), keystorePassword, + keystorePassword, System.getProperty("ssl.KeyManagerFactory.algorithm", KeyManagerFactory.getDefaultAlgorithm()), + System.getProperty("ssl.TrustManagerFactory.algorithm", TrustManagerFactory.getDefaultAlgorithm())); } } return KeyConfig.NONE; @@ -234,7 +235,7 @@ private static TrustConfig createCertChainTrustConfig(Settings settings, KeyConf return new StoreTrustConfig(trustStorePath, trustStoreType, trustStorePassword, trustStoreAlgorithm); } else if (global == null && System.getProperty("javax.net.ssl.trustStore") != null) { try (SecureString truststorePassword = new SecureString(System.getProperty("javax.net.ssl.trustStorePassword", ""))) { - return new StoreTrustConfig(System.getProperty("javax.net.ssl.trustStore"), "jks", truststorePassword, + return new StoreTrustConfig(System.getProperty("javax.net.ssl.trustStore"), KeyStore.getDefaultType(), truststorePassword, System.getProperty("ssl.TrustManagerFactory.algorithm", TrustManagerFactory.getDefaultAlgorithm())); } } else if (global != null && keyConfig == global.keyConfig()) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java index 63a5be610433b..0946ad3ac51c0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java @@ -194,7 +194,8 @@ public void testReloadingTrustStore() throws Exception { Path trustStorePath = tempDir.resolve("testnode.jks"); Path updatedTruststorePath = tempDir.resolve("testnode_updated.jks"); Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"), trustStorePath); - Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.jks"), updatedTruststorePath); + Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.jks"), + updatedTruststorePath); MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("xpack.ssl.truststore.secure_password", "testnode"); Settings settings = Settings.builder() From d8b3f332ef2adcfc7bc80b87189f08f1e1f6d655 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 29 Jun 2018 13:52:31 +0200 Subject: [PATCH 17/18] Remove extra check for object existence in repository-gcs read object (#31661) --- .../gcs/GoogleCloudStorageBlobStore.java | 16 +++++++++----- .../repositories/gcs/MockStorage.java | 22 ++++++++++++++++++- .../ESBlobStoreContainerTestCase.java | 6 ++++- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index c20b99790088e..2bfb3afc7d36c 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_PRECON_FAILED; class GoogleCloudStorageBlobStore extends AbstractComponent implements BlobStore { @@ -163,16 +164,19 @@ boolean blobExists(String blobName) throws IOException { */ InputStream readBlob(String blobName) throws IOException { final BlobId blobId = BlobId.of(bucketName, blobName); - final Blob blob = SocketAccess.doPrivilegedIOException(() -> client().get(blobId)); - if (blob == null) { - throw new NoSuchFileException("Blob [" + blobName + "] does not exit"); - } - final ReadChannel readChannel = SocketAccess.doPrivilegedIOException(blob::reader); + final ReadChannel readChannel = SocketAccess.doPrivilegedIOException(() -> client().reader(blobId)); return Channels.newInputStream(new ReadableByteChannel() { @SuppressForbidden(reason = "Channel is based of a socket not a file") @Override public int read(ByteBuffer dst) throws IOException { - return SocketAccess.doPrivilegedIOException(() -> readChannel.read(dst)); + try { + return SocketAccess.doPrivilegedIOException(() -> readChannel.read(dst)); + } catch (StorageException e) { + if (e.getCode() == HTTP_NOT_FOUND) { + throw new NoSuchFileException("Blob [" + blobName + "] does not exist"); + } + throw e; + } } @Override diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java index 605d1798ee826..cf7395ea1f1f7 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java @@ -167,7 +167,27 @@ public Iterable getValues() { public ReadChannel reader(BlobId blob, BlobSourceOption... options) { if (bucketName.equals(blob.getBucket())) { final byte[] bytes = blobs.get(blob.getName()); - final ReadableByteChannel readableByteChannel = Channels.newChannel(new ByteArrayInputStream(bytes)); + + final ReadableByteChannel readableByteChannel; + if (bytes != null) { + readableByteChannel = Channels.newChannel(new ByteArrayInputStream(bytes)); + } else { + readableByteChannel = new ReadableByteChannel() { + @Override + public int read(ByteBuffer dst) throws IOException { + throw new StorageException(404, "Object not found"); + } + + @Override + public boolean isOpen() { + return false; + } + + @Override + public void close() throws IOException { + } + }; + } return new ReadChannel() { @Override public void close() { diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java index 13f9e9debc966..43a62bbe662cc 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java @@ -49,7 +49,11 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase { public void testReadNonExistingPath() throws IOException { try(BlobStore store = newBlobStore()) { final BlobContainer container = store.blobContainer(new BlobPath()); - expectThrows(NoSuchFileException.class, () -> container.readBlob("non-existing")); + expectThrows(NoSuchFileException.class, () -> { + try (InputStream is = container.readBlob("non-existing")) { + is.read(); + } + }); } } From 8fa06294abe5f861db55c2cee04053c2fe285655 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 29 Jun 2018 14:15:34 +0200 Subject: [PATCH 18/18] Do not check for object existence when deleting repository index files (#31680) Before deleting a repository index generation file, BlobStoreRepository checks for the existence of the file and then deletes it. We can save a request here by using BlobContainer.deleteBlobIgnoringIfNotExists() which ignores error when deleting a file that does not exist. Since there is no way with S3 to know if a non versioned file existed before being deleted, this pull request also changes S3BlobContainer so that it now implements deleteBlobIgnoringIfNotExists(). It will now save one more request (blobExist?) when appropriate. The tests and fixture have been modified to conform the S3 API that always returns a 204/NO CONTENT HTTP response on deletions. --- .../repositories/s3/AmazonS3Fixture.java | 6 ++---- .../repositories/s3/S3BlobContainer.java | 5 +++++ .../elasticsearch/repositories/s3/MockAmazonS3.java | 10 ++-------- .../blobstore/BlobStoreIndexShardSnapshot.java | 13 ------------- .../repositories/blobstore/BlobStoreRepository.java | 12 +++--------- 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 20e21675acb79..d1034aff48248 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -174,10 +174,8 @@ private static PathTrie defaultHandlers(final Map clientReference.client().deleteObject(blobStore.bucket(), buildKey(blobName))); } catch (final AmazonClientException e) { throw new IOException("Exception when deleting blob [" + blobName + "]", e); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index d610e6d74a06d..b5fb01869ae8c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -149,15 +149,9 @@ public ObjectListing listObjects(final ListObjectsRequest request) throws Amazon @Override public void deleteObject(final DeleteObjectRequest request) throws AmazonClientException { assertThat(request.getBucketName(), equalTo(bucket)); - - final String blobName = request.getKey(); - if (blobs.remove(blobName) == null) { - AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist."); - exception.setStatusCode(404); - throw exception; - } + blobs.remove(request.getKey()); } - + @Override public void shutdown() { // TODO check close diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 275bc432942d3..297c12744cc26 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -389,19 +389,6 @@ public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List= 0) { final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2); - if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) { - snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile); - } + snapshotsBlobContainer.deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile); } // write the current generation to the index-latest file @@ -679,9 +677,7 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep bStream.writeLong(newGen); genBytes = bStream.bytes(); } - if (snapshotsBlobContainer.blobExists(INDEX_LATEST_BLOB)) { - snapshotsBlobContainer.deleteBlob(INDEX_LATEST_BLOB); - } + snapshotsBlobContainer.deleteBlobIgnoringIfNotExists(INDEX_LATEST_BLOB); logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); writeAtomic(INDEX_LATEST_BLOB, genBytes); } @@ -702,9 +698,7 @@ void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOExceptio } bytes = bStream.bytes(); } - if (snapshotsBlobContainer.blobExists(INCOMPATIBLE_SNAPSHOTS_BLOB)) { - snapshotsBlobContainer.deleteBlob(INCOMPATIBLE_SNAPSHOTS_BLOB); - } + snapshotsBlobContainer.deleteBlobIgnoringIfNotExists(INCOMPATIBLE_SNAPSHOTS_BLOB); // write the incompatible snapshots blob writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes); }