Autogenerate and print elastic pwd on startup

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialPasswordTests.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.apache.http.client.fluent.Request;
+import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.ServerUtils;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.BeforeClass;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.packaging.util.Archives.installArchive;
+import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assume.assumeTrue;
+
+public class ArchiveGenerateInitialPasswordTests extends PackagingTestCase {
+
+    private static final Pattern PASSWORD_REGEX = Pattern.compile("Password for the (\\w+) user is: (.+)$", Pattern.MULTILINE);
+
+    @BeforeClass
+    public static void filterDistros() {
+        assumeTrue("archives only", distribution.isArchive());
+    }
+
+    public void test10Install() throws Exception {
+        installation = installArchive(sh, distribution());
+        // Enable security for these tests only where it is necessary, until we can enable it for all
+        ServerUtils.enableSecurityFeatures(installation);
+        verifyArchiveInstallation(installation, distribution());
+    }
+
+    public void test20NoAutoGenerationWhenBootstrapPassword() throws Exception {
+        /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
+        assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
+        installation.executables().keystoreTool.run("create");
+        installation.executables().keystoreTool.run("add --stdin bootstrap.password", "some-password-here");
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+        String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), "elastic", "some-password-here", null);
+        assertThat(response, containsString("You Know, for Search"));
+    }
+
+    public void test30NoAutoGenerationWhenAutoConfigurationDisabled() throws Exception {
+        /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
+        assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
+        stopElasticsearch();
+        installation.executables().keystoreTool.run("remove bootstrap.password");
+        ServerUtils.disableSecurityAutoConfiguration(installation);
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+    }
+
+    public void test40VerifyAutogeneratedCredentials() throws Exception {
+        /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
+        assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
+        stopElasticsearch();
+        ServerUtils.enableSecurityAutoConfiguration(installation);
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.size(), equalTo(2));
+        assertThat(usersAndPasswords.containsKey("elastic"), is(true));
+        assertThat(usersAndPasswords.containsKey("kibana_system"), is(true));
+        for (Map.Entry<String, String> userpass : usersAndPasswords.entrySet()) {
+            String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
+            assertThat(response, containsString("You Know, for Search"));
+        }
+    }
+
+    public void test50PasswordAutogenerationOnlyOnce() throws Exception {
+        /* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
+        assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
+        stopElasticsearch();
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+    }
+
+    private Map<String, String> parseUsersAndPasswords(String output) {
+        Matcher matcher = PASSWORD_REGEX.matcher(output);
+        assertNotNull(matcher);
+        Map<String, String> usersAndPasswords = new HashMap<>();
+        while (matcher.find()) {
+            usersAndPasswords.put(matcher.group(1), matcher.group(2));
+        }
+        return usersAndPasswords;
+    }
+}

qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -20,8 +20,6 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
 import static java.nio.file.StandardOpenOption.APPEND;
@@ -119,16 +117,8 @@ public void test40RunWithCert() throws Exception {
     }
 
     private String setElasticPassword() {
-        final Pattern userpassRegex = Pattern.compile("PASSWORD (\\w+) = ([^\\s]+)");
-        Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
-        Matcher matcher = userpassRegex.matcher(result.stdout);
-        assertNotNull(matcher);
-        while (matcher.find()) {
-            if (matcher.group(1).equals("elastic")) {
-                return matcher.group(2);
-            }
-        }
-        return null;
+        Shell.Result result = installation.executables().resetElasticPasswordTool.run("--auto --batch --silent", null);
+        return result.stdout;
     }
 
 }

qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -41,6 +41,8 @@ public void test010Install() throws Exception {
         install();
         // Enable security for this test only where it is necessary, until we can enable it for all
         ServerUtils.enableSecurityFeatures(installation);
+        // Disable auto-configuration so that we can run setup-passwords
+        ServerUtils.disableSecurityAutoConfiguration(installation);
     }
 
     public void test20GeneratePasswords() throws Exception {

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -187,6 +187,7 @@ public class Executables {
         public final Executable nodeTool = new Executable("elasticsearch-node");
         public final Executable securityConfigTool = new Executable("elasticsearch-security-config");
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
+        public final Executable resetElasticPasswordTool = new Executable("elasticsearch-reset-elastic-password");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");
         public final Executable usersTool = new Executable("elasticsearch-users");

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -333,4 +333,26 @@ public static void enableSecurityFeatures(Installation installation) throws IOEx
         }
         Files.write(yml, lines, TRUNCATE_EXISTING);
     }
+
+    public static void disableSecurityAutoConfiguration(Installation installation) throws IOException {
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: false");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
+            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+        }
+        lines.addAll(addedLines);
+        Files.write(yml, lines, TRUNCATE_EXISTING);
+    }
+
+    public static void enableSecurityAutoConfiguration(Installation installation) throws IOException {
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: true");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
+            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+        }
+        lines.addAll(addedLines);
+        Files.write(yml, lines, TRUNCATE_EXISTING);
+    }
 }

qa/remote-clusters/docker-compose.yml

@@ -30,6 +30,7 @@ services:
        - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/testnode.crt
        - xpack.http.ssl.verification_mode=certificate
        - xpack.security.transport.ssl.verification_mode=certificate
+       - xpack.security.autoconfiguration.enabled=false
        - xpack.license.self_generated.type=trial
        - action.destructive_requires_name=false
     volumes:
@@ -84,6 +85,7 @@ services:
        - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/testnode.crt
        - xpack.http.ssl.verification_mode=certificate
        - xpack.security.transport.ssl.verification_mode=certificate
+       - xpack.security.autoconfiguration.enabled=false
        - xpack.license.self_generated.type=trial
        - action.destructive_requires_name=false
     volumes:

qa/system-indices/build.gradle

@@ -25,5 +25,6 @@ tasks.named("javaRestTest").configure {
 testClusters.all {
   testDistribution = 'DEFAULT'
   setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.autoconfiguration.enabled', 'false'
   user username: 'rest_user', password: 'rest-user-password', role: 'superuser'
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -105,6 +105,9 @@ private XPackSettings() {
     public static final Setting<Boolean> ENROLLMENT_ENABLED =
         Setting.boolSetting("xpack.security.enrollment.enabled", false, Property.NodeScope);
 
+    public static final Setting<Boolean> SECURITY_AUTOCONFIGURATION_ENABLED =
+        Setting.boolSetting("xpack.security.autoconfiguration.enabled", true, Property.NodeScope);
+
     /*
      * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them
      * but instead parse based on a prefix (eg *.ssl.*)
@@ -232,6 +235,7 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(USER_SETTING);
         settings.add(PASSWORD_HASHING_ALGORITHM);
         settings.add(ENROLLMENT_ENABLED);
+        settings.add(SECURITY_AUTOCONFIGURATION_ENABLED);
         return Collections.unmodifiableList(settings);
     }
 

x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle

@@ -16,6 +16,7 @@ testClusters {
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.watcher.enabled', 'false'
     setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.autoconfiguration.enabled', 'false'
 
     user username: "test_user", password: "x-pack-test-password"
   }
@@ -29,6 +30,7 @@ testClusters {
     }
     setting 'cluster.remote.connections_per_cluster', "1"
     setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.autoconfiguration.enabled', 'false'
 
     user username: "test_user", password: "x-pack-test-password"
   }

x-pack/plugin/fleet/build.gradle

@@ -26,5 +26,6 @@ dependencies {
 testClusters.all {
   testDistribution = 'DEFAULT'
   setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.autoconfiguration.enabled', 'false'
   user username: 'x_pack_rest_user', password: 'x-pack-test-password', role: 'superuser'
 }

x-pack/plugin/ilm/qa/rest/build.gradle

@@ -32,5 +32,6 @@ testClusters.all {
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.autoconfiguration.enabled', 'false'
   user clusterCredentials
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/GenerateInitialBuiltinUsersPasswordListener.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.xpack.core.security.user.ElasticUser;
+import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
+import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+
+import java.util.function.BiConsumer;
+
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
+
+public class GenerateInitialBuiltinUsersPasswordListener implements BiConsumer<SecurityIndexManager.State, SecurityIndexManager.State> {
+
+    private static final Logger LOGGER  = LogManager.getLogger(GenerateInitialBuiltinUsersPasswordListener.class);
+    private NativeUsersStore nativeUsersStore;
+    private SecurityIndexManager securityIndexManager;
+
+    public GenerateInitialBuiltinUsersPasswordListener(NativeUsersStore nativeUsersStore, SecurityIndexManager securityIndexManager) {
+        this.nativeUsersStore = nativeUsersStore;
+        this.securityIndexManager = securityIndexManager;
+    }
+
+    @Override
+    public void accept(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) {
+        if (previousState.equals(SecurityIndexManager.State.UNRECOVERED_STATE)
+            && currentState.equals(SecurityIndexManager.State.UNRECOVERED_STATE) == false
+            && securityIndexManager.indexExists() == false) {
+
+            final SecureString elasticPassword = new SecureString(generatePassword(20));
+            final SecureString kibanaSystemPassword = new SecureString(generatePassword(20));
+            nativeUsersStore
+                .updateReservedUser(
+                    ElasticUser.NAME,
+                    elasticPassword.getChars(),
+                    DocWriteRequest.OpType.CREATE,
+                    WriteRequest.RefreshPolicy.IMMEDIATE,
+                    ActionListener.wrap(result -> {
+                        nativeUsersStore
+                            .updateReservedUser(
+                                KibanaSystemUser.NAME,
+                                kibanaSystemPassword.getChars(),
+                                DocWriteRequest.OpType.CREATE,
+                                WriteRequest.RefreshPolicy.IMMEDIATE,
+                                ActionListener.wrap(
+                                    r -> {
+                                        outputOnSuccess(elasticPassword, kibanaSystemPassword);
+                                    }, this::outputOnError
+                                )
+                            );
+                    }, this::outputOnError));
+            securityIndexManager.removeStateListener(this);
+        }
+    }
+
+    private void outputOnSuccess(SecureString elasticPassword, SecureString kibanaSystemPassword) {
+        LOGGER.info("");
+        LOGGER.info("-----------------------------------------------------------------");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("Password for the elastic user is: " + elasticPassword);
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("Password for the kibana_system user is: " + kibanaSystemPassword);
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("Please note these down as they will not be shown again.");
+        LOGGER.info("");
+        LOGGER.info("You can use 'bin/elasticsearch-reset-elastic-password' at any time");
+        LOGGER.info("in order to reset the password for the elastic user.");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("You can use 'bin/elasticsearch-reset-kibana-system-password' at any time");
+        LOGGER.info("in order to reset the password for the kibana_system user.");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("");
+        LOGGER.info("-----------------------------------------------------------------");
+        LOGGER.info("");
+    }
+
+    private void outputOnError(Exception e) {
+        if (e instanceof VersionConflictEngineException == false) {
+            LOGGER.info("");
+            LOGGER.info("-----------------------------------------------------------------");
+            LOGGER.info("");
+            LOGGER.info("");
+            LOGGER.info("");
+            LOGGER.info("Failed to set the password for the elastic and kibana-system users ");
+            LOGGER.info("automatically");
+            LOGGER.info("");
+            LOGGER.info("You can use 'bin/elasticsearch-reset-elastic-password'");
+            LOGGER.info("in order to set the password for the elastic user.");
+            LOGGER.info("");
+            LOGGER.info("");
+            LOGGER.info("You can use 'bin/elasticsearch-reset-kibana-system-password'");
+            LOGGER.info("in order to set the password for the kibana_system user.");
+            LOGGER.info("");
+            LOGGER.info("");
+            LOGGER.info("");
+            LOGGER.info("-----------------------------------------------------------------");
+            LOGGER.info("");
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -337,8 +337,10 @@
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
+import static org.elasticsearch.xpack.core.XPackSettings.SECURITY_AUTOCONFIGURATION_ENABLED;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS;
+import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.BOOTSTRAP_ELASTIC_PASSWORD;
 import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_TOKENS_INDEX_FORMAT;
@@ -453,7 +455,6 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         }
 
         scriptServiceReference.set(scriptService);
-
         // We need to construct the checks here while the secure settings are still available.
         // If we wait until #getBoostrapChecks the secure settings will have been cleared/closed.
         final List<BootstrapCheck> checks = new ArrayList<>();
@@ -495,6 +496,11 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
 
         // realms construction
         final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client, securityIndex.get());
+        GenerateInitialBuiltinUsersPasswordListener generateInitialBuiltinUsersPasswordListener =
+            new GenerateInitialBuiltinUsersPasswordListener(nativeUsersStore, securityIndex.get());
+        if (BOOTSTRAP_ELASTIC_PASSWORD.exists(settings) == false && SECURITY_AUTOCONFIGURATION_ENABLED.get(settings)) {
+            securityIndex.get().addStateListener(generateInitialBuiltinUsersPasswordListener);
+        }
         final NativeRoleMappingStore nativeRoleMappingStore = new NativeRoleMappingStore(settings, client, securityIndex.get(),
             scriptService);
         final AnonymousUser anonymousUser = new AnonymousUser(settings);