Autogenerate and print elastic pwd on startup

qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -20,8 +20,6 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeFalse;
 import static java.nio.file.StandardOpenOption.APPEND;
@@ -119,16 +117,8 @@ public void test40RunWithCert() throws Exception {
     }
 
     private String setElasticPassword() {
-        final Pattern userpassRegex = Pattern.compile("PASSWORD (\\w+) = ([^\\s]+)");
-        Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
-        Matcher matcher = userpassRegex.matcher(result.stdout);
-        assertNotNull(matcher);
-        while (matcher.find()) {
-            if (matcher.group(1).equals("elastic")) {
-                return matcher.group(2);
-            }
-        }
-        return null;
+        Shell.Result result = installation.executables().resetElasticPasswordTool.run("--auto --batch --silent", null);
+        return result.stdout;
     }
 
 }

qa/os/src/test/java/org/elasticsearch/packaging/test/GenerateInitialPasswordTests.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.apache.http.client.fluent.Request;
+import org.elasticsearch.packaging.util.ServerUtils;
+import org.elasticsearch.packaging.util.Shell;
+import org.junit.BeforeClass;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.packaging.util.Archives.installArchive;
+import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assume.assumeTrue;
+
+public class GenerateInitialPasswordTests extends PackagingTestCase {
+
+    private static final Pattern PASSWORD_REGEX = Pattern.compile("Password for the (\\w+) user is: (.+)$");
+
+    @BeforeClass
+    public static void filterDistros() {
+        assumeTrue("archives and docker only", distribution.isArchive() || distribution.isDocker());
+    }
+
+    public void test10Install() throws Exception {
+        installation = installArchive(sh, distribution());
+        // Enable security for these tests only where it is necessary, until we can enable it for all
+        ServerUtils.enableSecurityFeatures(installation);
+        verifyArchiveInstallation(installation, distribution());
+
+    }
+
+    public void test20NoAutoGenerationWhenBootstrapPassword() throws Exception {
+        installation.executables().keystoreTool.run("add --stdin bootstrap.password", "some-password-here");
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+        String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), "elastic", "some-password-here", null);
+        assertThat(response, containsString("You Know, for Search"));
+    }
+
+    public void test30NoAutoGenerationWhenAutoConfigurationDisabled() throws Exception {
+        stopElasticsearch();
+        installation.executables().keystoreTool.run("remove bootstrap.password");
+        ServerUtils.disableSecurityAutoConfiguration(installation);
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+    }
+
+    public void test40VerifyAutogeneratedCredentials() throws Exception {
+        stopElasticsearch();
+        ServerUtils.enableSecurityAutoConfiguration(installation);
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.size(), equalTo(2));
+        assertThat(usersAndPasswords.containsKey("elastic"), is(true));
+        assertThat(usersAndPasswords.containsKey("kibana_system"), is(true));
+        for (Map.Entry<String, String> userpass : usersAndPasswords.entrySet()) {
+            String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue(), null);
+            assertThat(response, containsString("You Know, for Search"));
+        }
+    }
+
+    public void test50PasswordAutogenerationOnlyOnce() throws Exception {
+        stopElasticsearch();
+        Shell.Result result = runElasticsearchStartCommand(null, false, true);
+        Map<String, String> usersAndPasswords = parseUsersAndPasswords(result.stdout);
+        assertThat(usersAndPasswords.isEmpty(), is(true));
+    }
+
+    private Map<String, String> parseUsersAndPasswords(String output) {
+        Matcher matcher = PASSWORD_REGEX.matcher(output);
+        assertNotNull(matcher);
+        Map<String, String> userpases = new HashMap<>();
+        while (matcher.find()) {
+            userpases.put(matcher.group(1), matcher.group(2));
+        }
+        return userpases;
+    }
+}

qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -41,6 +41,8 @@ public void test010Install() throws Exception {
         install();
         // Enable security for this test only where it is necessary, until we can enable it for all
         ServerUtils.enableSecurityFeatures(installation);
+        // Disable auto-configuration so that we can run setup-passwords
+        ServerUtils.disableSecurityAutoConfiguration(installation);
     }
 
     public void test20GeneratePasswords() throws Exception {

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -187,6 +187,7 @@ public class Executables {
         public final Executable nodeTool = new Executable("elasticsearch-node");
         public final Executable securityConfigTool = new Executable("elasticsearch-security-config");
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
+        public final Executable resetElasticPasswordTool = new Executable("elasticsearch-reset-elastic-password");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");
         public final Executable usersTool = new Executable("elasticsearch-users");

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -333,4 +333,26 @@ public static void enableSecurityFeatures(Installation installation) throws IOEx
         }
         Files.write(yml, lines, TRUNCATE_EXISTING);
     }
+
+    public static void disableSecurityAutoConfiguration(Installation installation) throws IOException {
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: false");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
+            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+        }
+        lines.addAll(addedLines);
+        Files.write(yml, lines, TRUNCATE_EXISTING);
+    }
+
+    public static void enableSecurityAutoConfiguration(Installation installation) throws IOException {
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> addedLines = List.of("xpack.security.autoconfiguration.enabled: true");
+        List<String> lines;
+        try (Stream<String> allLines = Files.readAllLines(yml).stream()) {
+            lines = allLines.filter(s -> s.startsWith("xpack.security.autoconfiguration.enabled") == false).collect(Collectors.toList());
+        }
+        lines.addAll(addedLines);
+        Files.write(yml, lines, TRUNCATE_EXISTING);
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -105,6 +105,9 @@ private XPackSettings() {
     public static final Setting<Boolean> ENROLLMENT_ENABLED =
         Setting.boolSetting("xpack.security.enrollment.enabled", false, Property.NodeScope);
 
+    public static final Setting<Boolean> SECURITY_AUTOCONFIGURATION_ENABLED =
+        Setting.boolSetting("xpack.security.autoconfiguration.enabled", true, Property.NodeScope);
+
     /*
      * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them
      * but instead parse based on a prefix (eg *.ssl.*)
@@ -232,6 +235,7 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(USER_SETTING);
         settings.add(PASSWORD_HASHING_ALGORITHM);
         settings.add(ENROLLMENT_ENABLED);
+        settings.add(SECURITY_AUTOCONFIGURATION_ENABLED);
         return Collections.unmodifiableList(settings);
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/GenerateInitialBuiltinUsersPasswordListener.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.xpack.core.security.user.ElasticUser;
+import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
+import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+
+import java.util.function.BiConsumer;
+
+import static org.elasticsearch.xpack.security.tool.CommandUtils.generatePassword;
+
+public class GenerateInitialBuiltinUsersPasswordListener implements BiConsumer<SecurityIndexManager.State, SecurityIndexManager.State> {
+
+    private static final Logger LOGGER  = LogManager.getLogger(GenerateInitialBuiltinUsersPasswordListener.class);
+    private NativeUsersStore nativeUsersStore;
+    private SecurityIndexManager securityIndexManager;
+
+    public GenerateInitialBuiltinUsersPasswordListener(NativeUsersStore nativeUsersStore, SecurityIndexManager securityIndexManager) {
+        this.nativeUsersStore = nativeUsersStore;
+        this.securityIndexManager = securityIndexManager;
+    }
+
+    @Override
+    public void accept(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) {
+        if (previousState.equals(SecurityIndexManager.State.UNRECOVERED_STATE)
+            && currentState.equals(SecurityIndexManager.State.UNRECOVERED_STATE) == false
+            && securityIndexManager.indexExists() == false) {
+
+            final SecureString elasticPassword = new SecureString(generatePassword(20));
+            final SecureString kibanaSystemPassword = new SecureString(generatePassword(20));
+            final GroupedActionListener<Void> changePasswordListener = new GroupedActionListener<>(
+                ActionListener.wrap(() -> {
+                    nativeUsersStore
+                        .updateReservedUser(
+                            KibanaSystemUser.NAME,
+                            kibanaSystemPassword.getChars(),
+                            DocWriteRequest.OpType.CREATE,
+                            WriteRequest.RefreshPolicy.IMMEDIATE,
+                            ActionListener.wrap(
+                                r -> {
+                                    LOGGER.info("");
+                                    LOGGER.info("-----------------------------------------------------------------");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("Password for the elastic user is: " + elasticPassword);
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("Password for the kibana user is: " + kibanaSystemPassword);
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("Please note these down as the will not be shown again.");
+                                    LOGGER.info("");
+                                    LOGGER.info("You can use 'bin/elasticsearch-reset-elastic-password' at any time");
+                                    LOGGER.info("in order to reset the password for the elastic user.");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("You can use 'bin/elasticsearch-reset-kibana-system-password' at any time");
+                                    LOGGER.info("in order to reset the password for the kibana-system user.");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("");
+                                    LOGGER.info("-----------------------------------------------------------------");
+                                    LOGGER.info("");
+                                },
+                                e -> {
+                                    if (e instanceof VersionConflictEngineException == false) {
+                                        LOGGER.info("");
+                                        LOGGER.info("-----------------------------------------------------------------");
+                                        LOGGER.info("");
+                                        LOGGER.info("");
+                                        LOGGER.info("");
+                                        LOGGER.info("Failed to set the password for the elastic and kibana-system users ");
+                                        LOGGER.info("automatically");
+                                        LOGGER.info("");
+                                        LOGGER.info("You can use 'bin/elasticsearch-reset-elastic-password'");
+                                        LOGGER.info("in order to set the password for the elastic user.");
+                                        LOGGER.info("");
+                                        LOGGER.info("");
+                                        LOGGER.info("You can use 'bin/elasticsearch-reset-kibana-system-password'");
+                                        LOGGER.info("in order to set the password for the kibana-system user.");
+                                        LOGGER.info("");
+                                        LOGGER.info("");
+                                        LOGGER.info("");
+                                        LOGGER.info("-----------------------------------------------------------------");
+                                        LOGGER.info("");
+                                    }
+                                    LOGGER.warn(e);
+                                }
+                            )
+                        );
+                }),1);
+            nativeUsersStore
+                .updateReservedUser(
+                    ElasticUser.NAME,
+                    elasticPassword.getChars(),
+                    DocWriteRequest.OpType.CREATE,
+                    WriteRequest.RefreshPolicy.IMMEDIATE,
+                    changePasswordListener);
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -337,8 +337,10 @@
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
+import static org.elasticsearch.xpack.core.XPackSettings.SECURITY_AUTOCONFIGURATION_ENABLED;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_TOKENS_ALIAS;
+import static org.elasticsearch.xpack.security.authc.esnative.ReservedRealm.BOOTSTRAP_ELASTIC_PASSWORD;
 import static org.elasticsearch.xpack.security.operator.OperatorPrivileges.OPERATOR_PRIVILEGES_ENABLED;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_TOKENS_INDEX_FORMAT;
@@ -390,6 +392,7 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
     private final List<SecurityExtension> securityExtensions = new ArrayList<>();
     private final SetOnce<Transport> transportReference = new SetOnce<>();
     private final SetOnce<ScriptService> scriptServiceReference = new SetOnce<>();
+    private final SetOnce<NativeUsersStore> nativeUsersStoreReference = new SetOnce<>();
 
     public Security(Settings settings, final Path configPath) {
         this(settings, configPath, Collections.emptyList());
@@ -453,7 +456,6 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         }
 
         scriptServiceReference.set(scriptService);
-
         // We need to construct the checks here while the secure settings are still available.
         // If we wait until #getBoostrapChecks the secure settings will have been cleared/closed.
         final List<BootstrapCheck> checks = new ArrayList<>();
@@ -495,6 +497,11 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
 
         // realms construction
         final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client, securityIndex.get());
+        GenerateInitialBuiltinUsersPasswordListener generateInitialBuiltinUsersPasswordListener =
+            new GenerateInitialBuiltinUsersPasswordListener(nativeUsersStore, securityIndex.get());
+        if (BOOTSTRAP_ELASTIC_PASSWORD.exists(settings) == false && SECURITY_AUTOCONFIGURATION_ENABLED.get(settings)) {
+            securityIndex.get().addStateListener(generateInitialBuiltinUsersPasswordListener);
+        }
         final NativeRoleMappingStore nativeRoleMappingStore = new NativeRoleMappingStore(settings, client, securityIndex.get(),
             scriptService);
         final AnonymousUser anonymousUser = new AnonymousUser(settings);