Auto-configure the elastic user password

server/src/main/java/org/elasticsearch/ElasticsearchAuthenticationProcessingError.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+/**
+ * Used to indicate that the authentication process encountered a server-side error (5xx) that prevented the credentials verification.
+ * The presented client credentials might or might not be valid.
+ * This differs from an authentication failure error in subtle ways. This should be preferred when the issue hindering credentials
+ * verification is transient, such as network congestion or overloaded instances, but not in cases of misconfiguration.
+ * However this distinction is further blurred because in certain configurations and for certain credential types, the same
+ * credential can be validated in multiple ways, only some of which might experience transient problems.
+ * When in doubt, rely on the implicit behavior of 401 authentication failure.
+ */
+public class ElasticsearchAuthenticationProcessingError extends ElasticsearchSecurityException {
+
+    public ElasticsearchAuthenticationProcessingError(String msg, RestStatus status, Throwable cause, Object... args) {
+        super(msg, status, cause, args);
+        assert status == RestStatus.INTERNAL_SERVER_ERROR || status == RestStatus.SERVICE_UNAVAILABLE;
+    }
+
+    public ElasticsearchAuthenticationProcessingError(StreamInput in) throws IOException {
+        super(in);
+    }
+}

server/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -1038,7 +1038,12 @@ private enum ElasticsearchExceptionHandle {
                 org.elasticsearch.action.search.VersionMismatchException.class,
                 org.elasticsearch.action.search.VersionMismatchException::new,
                 161,
-                Version.V_7_12_0);
+                Version.V_7_12_0),
+        AUTHENTICATION_PROCESSING_ERROR(
+            org.elasticsearch.ElasticsearchAuthenticationProcessingError.class,
+            org.elasticsearch.ElasticsearchAuthenticationProcessingError::new,
+            162,
+            Version.V_7_16_0);
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;

server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java

@@ -822,6 +822,7 @@ public void testIds() {
         ids.put(159, NodeHealthCheckFailureException.class);
         ids.put(160, NoSeedNodeLeftException.class);
         ids.put(161, VersionMismatchException.class);
+        ids.put(162, ElasticsearchAuthenticationProcessingError.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java

@@ -134,6 +134,20 @@ public static AuthenticationResult terminate(String message, @Nullable Exception
         return new AuthenticationResult(Status.TERMINATE, null, message, cause, null);
     }
 
+    /**
+     * Creates an {@code AuthenticationResult} that indicates that the realm attempted to handle the authentication request, was
+     * unsuccessful and wants to terminate this authentication request.
+     * The reason for the failure is given in the supplied message.
+     * <p>
+     * The {@link #getStatus() status} is set to {@link Status#TERMINATE}.
+     * </p><p>
+     * The {@link #getUser() user} is not populated.
+     * </p>
+     */
+    public static AuthenticationResult terminate(String message) {
+        return terminate(message, null);
+    }
+
     public boolean isAuthenticated() {
         return status == Status.SUCCESS;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc;
 
+import org.elasticsearch.ElasticsearchAuthenticationProcessingError;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestRequest;
@@ -100,12 +101,22 @@ public ElasticsearchSecurityException failedAuthentication(TransportMessage mess
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(RestRequest request, Exception e, ThreadContext context) {
+        // a couple of authn processing errors can also return {@link RestStatus#INTERNAL_SERVER_ERROR} or
+        // {@link RestStatus#SERVICE_UNAVAILABLE}, besides the obvious {@link RestStatus#UNAUTHORIZED}
+        if (e instanceof ElasticsearchAuthenticationProcessingError) {
+            return (ElasticsearchAuthenticationProcessingError) e;
+        }
         return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(TransportMessage message, String action, Exception e,
             ThreadContext context) {
+        // a couple of authn processing errors can also return {@link RestStatus#INTERNAL_SERVER_ERROR} or
+        // {@link RestStatus#SERVICE_UNAVAILABLE}, besides the obvious {@link RestStatus#UNAUTHORIZED}
+        if (e instanceof ElasticsearchAuthenticationProcessingError) {
+            return (ElasticsearchAuthenticationProcessingError) e;
+        }
         return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java

@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.core.security.support;
 
+import org.elasticsearch.ElasticsearchAuthenticationProcessingError;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.XPackField;
 
@@ -34,4 +36,12 @@ public static ElasticsearchSecurityException authorizationError(String msg, Obje
     public static ElasticsearchSecurityException authorizationError(String msg, Exception cause, Object... args) {
         return new ElasticsearchSecurityException(msg, RestStatus.FORBIDDEN, cause, args);
     }
+
+    public static ElasticsearchAuthenticationProcessingError authenticationProcessError(String msg, Exception cause, Object... args) {
+        RestStatus restStatus = RestStatus.SERVICE_UNAVAILABLE;
+        if (RestStatus.INTERNAL_SERVER_ERROR == ExceptionsHelper.status(ExceptionsHelper.unwrapCause(cause))) {
+            restStatus = RestStatus.INTERNAL_SERVER_ERROR;
+        }
+        return new ElasticsearchAuthenticationProcessingError(msg, restStatus, cause, args);
+    }
 }

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java

@@ -58,12 +58,6 @@ public abstract class SecuritySingleNodeTestCase extends ESSingleNodeTestCase {
     private static SecuritySettingsSource SECURITY_DEFAULT_SETTINGS = null;
     private static CustomSecuritySettingsSource customSecuritySettingsSource = null;
     private static RestClient restClient = null;
-    private static SecureString BOOTSTRAP_PASSWORD = null;
-
-    @BeforeClass
-    public static void generateBootstrapPassword() {
-        BOOTSTRAP_PASSWORD = TEST_PASSWORD_SECURE_STRING.clone();
-    }
 
     @BeforeClass
     public static void initDefaultSettings() {
@@ -82,10 +76,6 @@ public static void initDefaultSettings() {
     public static void destroyDefaultSettings() {
         SECURITY_DEFAULT_SETTINGS = null;
         customSecuritySettingsSource = null;
-        if (BOOTSTRAP_PASSWORD != null) {
-            BOOTSTRAP_PASSWORD.close();
-            BOOTSTRAP_PASSWORD = null;
-        }
         tearDownRestClient();
     }
 
@@ -180,10 +170,17 @@ protected Settings nodeSettings() {
         if (builder.getSecureSettings() == null) {
             builder.setSecureSettings(new MockSecureSettings());
         }
-        ((MockSecureSettings) builder.getSecureSettings()).setString("bootstrap.password", BOOTSTRAP_PASSWORD.toString());
+        SecureString boostrapPassword = getBootstrapPassword();
+        if (boostrapPassword != null) {
+            ((MockSecureSettings) builder.getSecureSettings()).setString("bootstrap.password", boostrapPassword.toString());
+        }
         return builder.build();
     }
 
+    protected SecureString getBootstrapPassword() {
+        return TEST_PASSWORD_SECURE_STRING;
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return customSecuritySettingsSource.nodePlugins();

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.esnative;
+
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.SecuritySingleNodeTestCase;
+import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
+import org.elasticsearch.xpack.core.security.action.user.PutUserRequest;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
+import org.junit.BeforeClass;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
+import static org.hamcrest.Matchers.is;
+
+public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNodeTestCase {
+
+    private static Hasher hasher;
+
+    @BeforeClass
+    public static void setHasher() {
+        hasher = getFastStoredHashAlgoForTests();
+    }
+
+    @Override
+    public Settings nodeSettings() {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(super.nodeSettings())
+            .put("xpack.security.authc.password_hashing.algorithm", hasher.name());
+        ((MockSecureSettings) settingsBuilder.getSecureSettings()).setString("autoconfiguration.password_hash",
+            new String(hasher.hash(new SecureString("auto_password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray()))));
+        return settingsBuilder.build();
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable HTTP
+    }
+
+    @Override
+    protected SecureString getBootstrapPassword() {
+        return null; // no bootstrap password for this test
+    }
+
+    public void testAutoconfigFailedPasswordPromotion() {
+        try {
+            // prevents the .security index from being created automatically (after elastic user authentication)
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(),
+                true));
+            assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+            // delete the security index, if it exist
+            GetIndexRequest getIndexRequest = new GetIndexRequest();
+            getIndexRequest.indices(SECURITY_MAIN_ALIAS);
+            getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+            GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet();
+            if (getIndexResponse.getIndices().length > 0) {
+                assertThat(getIndexResponse.getIndices().length, is(1));
+                assertThat(getIndexResponse.getIndices()[0], is(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7));
+                DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices());
+                assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet());
+            }
+
+            // elastic user gets 503 for the good password
+            Request restRequest = randomFrom(new Request("GET", "/_security/_authenticate"), new Request("GET", "_cluster/health"),
+                new Request("GET", "_nodes"));
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue("elastic",
+                new SecureString("auto_password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray())));
+            restRequest.setOptions(options);
+            ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(restRequest));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(RestStatus.SERVICE_UNAVAILABLE.getStatus()));
+
+            // but gets a 401 for the wrong password
+            Request restRequest2 = randomFrom(new Request("GET", "/_security/_authenticate"), new Request("GET", "_cluster/health"),
+                new Request("GET", "_nodes"));
+            options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue("elastic",
+                new SecureString("wrong password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray())));
+            restRequest2.setOptions(options);
+            exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(restRequest2));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(RestStatus.UNAUTHORIZED.getStatus()));
+        } finally {
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(),
+                (String) null));
+            assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        }
+    }
+
+    public void testAutoconfigSucceedsAfterPromotionFailure() throws Exception {
+        try {
+            // create any non-elastic user, which triggers .security index creation
+            final PutUserRequest putUserRequest = new PutUserRequest();
+            final String username = randomAlphaOfLength(8);
+            putUserRequest.username(username);
+            final SecureString password = new SecureString("super-strong-password!".toCharArray());
+            putUserRequest.passwordHash(Hasher.PBKDF2.hash(password));
+            putUserRequest.roles(Strings.EMPTY_ARRAY);
+            client().execute(PutUserAction.INSTANCE, putUserRequest).get();
+
+            // but then make the cluster read-only
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(),
+                true));
+            assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+            // elastic user now gets 503 for the good password
+            Request restRequest = randomFrom(new Request("GET", "/_security/_authenticate"), new Request("GET", "_cluster/health"),
+                new Request("GET", "_nodes"));
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue("elastic",
+                new SecureString("auto_password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray())));
+            restRequest.setOptions(options);
+            ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(restRequest));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(RestStatus.SERVICE_UNAVAILABLE.getStatus()));
+
+            // clear cluster-wide write block
+            updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(),
+                (String) null));
+            assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+            if (randomBoolean()) {
+                Request restRequest2 = randomFrom(new Request("GET", "/_security/_authenticate"), new Request("GET", "_cluster/health"),
+                    new Request("GET", "_nodes"));
+                options = RequestOptions.DEFAULT.toBuilder();
+                options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue("elastic",
+                    new SecureString("wrong password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray())));
+                restRequest2.setOptions(options);
+                exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(restRequest2));
+                assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(RestStatus.UNAUTHORIZED.getStatus()));
+            }
+
+            // now the auto config password can be promoted, and authn succeeds
+            Request restRequest3 = randomFrom(new Request("GET", "/_security/_authenticate"), new Request("GET", "_cluster/health"),
+                new Request("GET", "_nodes"));
+            options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue("elastic",
+                new SecureString("auto_password_that_is_longer_than_14_chars_because_of_FIPS".toCharArray())));
+            restRequest3.setOptions(options);
+            assertThat(getRestClient().performRequest(restRequest3).getStatusLine().getStatusCode(), is(RestStatus.OK.getStatus()));
+        } finally {
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(),
+                (String) null));
+            assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialSecurityConfigurationListener.java

@@ -31,7 +31,7 @@
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH;
+import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.AUTOCONFIG_ELASTIC_PASSWORD_HASH;
 import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.BOOTSTRAP_ELASTIC_PASSWORD;
 import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
 
@@ -87,7 +87,7 @@ public void accept(SecurityIndexManager.State previousState, SecurityIndexManage
             }, this::outputOnError), 2);
 
             if (false == BOOTSTRAP_ELASTIC_PASSWORD.exists(environment.settings())
-                && false == AUTOCONFIG_BOOOTSTRAP_ELASTIC_PASSWORD_HASH.exists(environment.settings())) {
+                && false == AUTOCONFIG_ELASTIC_PASSWORD_HASH.exists(environment.settings())) {
                 final SecureString elasticPassword = new SecureString(generatePassword(20));
                 nativeUsersStore.updateReservedUser(
                     ElasticUser.NAME,