Security auto-configuration for packaged installations

distribution/packages/build.gradle

@@ -254,7 +254,7 @@ def commonPackageConfig(String type, String architecture) {
   }
 }
 
-// this is package indepdendent configuration
+// this is package independent configuration
 ospackage {
   maintainer 'Elasticsearch Team <[email protected]>'
   summary 'Distributed RESTful search engine built for the cloud'

distribution/packages/src/common/scripts/postinst

@@ -49,20 +49,67 @@ case "$1" in
         exit 1
     ;;
 esac
-
 # to pick up /usr/lib/sysctl.d/elasticsearch.conf
 if command -v systemctl > /dev/null; then
     systemctl restart systemd-sysctl.service || true
 fi
-
 if [ "x$IS_UPGRADE" != "xtrue" ]; then
+    # Don't exit immediately on error, we want to hopefully print some helpful banners
+    set +e
+    # Attempt to auto-configure security, this seems to be an installation
+    if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
+    ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+    ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+    /usr/share/elasticsearch/bin/elasticsearch-cli <<< ""; then
+        # Above command runs as root and TLS keystores are created group-owned by root. It's simple to correct the ownership here
+        for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_*
+        do
+            chown root:elasticsearch "${dir}"/http_keystore_local_node.p12
+            chown root:elasticsearch "${dir}"/http_ca.crt
+            chown root:elasticsearch "${dir}"/transport_keystore_all_nodes.p12
+        done
+        if INITIAL_PASSWORD=$(ES_MAIN_CLASS=org.elasticsearch.xpack.security.enrollment.tool.AutoConfigGenerateElasticPasswordHash \
+        ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+        ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+        /usr/share/elasticsearch/bin/elasticsearch-cli); then
+            echo "##########         Security autoconfiguration information         ############"
+            echo "#                                                                            #"
+            echo "# Authentication and Authorization are enabled.                              #"
+            echo "# TLS for the transport and the http layers is enabled and configured.       #"
+            echo "#                                                                            #"
+            echo "# The password of the elastic superuser will be set to: ${INITIAL_PASSWORD} #"
+            echo "# upon starting elasticsearch for the first time                             #"
+            echo "#                                                                            #"
+            echo "##############################################################################"
+        fi
+    else
+        if [ $? -eq 80 ]; then
+            # ExitCodes.NOOP
+            echo "##########         Security autoconfiguration information         ############"
+            echo "#                                                                            #"
+            echo "# Security features appear to be already configured.                         #"
+            echo "#                                                                            #"
+            echo "##############################################################################"
+        else
+            echo "##########         Security autoconfiguration information         ############"
+            echo "#                                                                            #"
+            echo "# Failed to auto-configure security features.                                #"
+            echo "# Authentication and Authorization are enabled.                              #"
+            echo "# You can use elasticsearch-reset-elastic-password to set a password         #"
+            echo "# for the elastic user.                                                      #"
+            echo "#                                                                            #"
+            echo "##############################################################################"
+        fi
+    fi
     if command -v systemctl >/dev/null; then
         echo "### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd"
         echo " sudo systemctl daemon-reload"
         echo " sudo systemctl enable elasticsearch.service"
         echo "### You can start elasticsearch service by executing"
         echo " sudo systemctl start elasticsearch.service"
     fi
+    set -e
+
 elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
 
     echo -n "Restarting elasticsearch service..."

distribution/packages/src/common/scripts/postrm

@@ -18,6 +18,7 @@ export ES_PATH_CONF=${ES_PATH_CONF:[email protected]@}
 
 REMOVE_DIRS=false
 REMOVE_JVM_OPTIONS_DIRECTORY=false
+REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY=false
 REMOVE_ELASTICSEARCH_KEYSTORE=false
 REMOVE_USER_AND_GROUP=false
 
@@ -31,6 +32,7 @@ case "$1" in
     purge)
         REMOVE_DIRS=true
         REMOVE_JVM_OPTIONS_DIRECTORY=true
+        REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY=true
         REMOVE_ELASTICSEARCH_KEYSTORE=true
         REMOVE_USER_AND_GROUP=true
     ;;
@@ -99,6 +101,16 @@ if [ "$REMOVE_DIRS" = "true" ]; then
       fi
     fi
 
+    # delete the security auto config directory if we are purging
+    if [ "$REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY" = "true" ]; then
+      for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_*
+      do
+        echo -n "Deleting security auto-configuration directory..."
+        rm -rf "${dir}"
+        echo "OK"
+      done
+    fi
+
     # delete the elasticsearch keystore if we are purging
     if [ "$REMOVE_ELASTICSEARCH_KEYSTORE" = "true" ]; then
       if [ -e "${ES_PATH_CONF}/elasticsearch.keystore" ]; then

docs/changelog/75144.yaml

@@ -0,0 +1,6 @@
+pr: 75144
+summary: Security auto-configuration for packaged installations
+area: Security
+type: enhancement
+issues:
+ - 78306

qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java

@@ -48,9 +48,6 @@ public static void cleanupFiles() {
 
     public void test10Install() throws Exception {
         install();
-        // Enable security for this test only where it is necessary, until we can enable it for all
-        // Only needed until https://github.com/elastic/elasticsearch/pull/75144 is merged
-        ServerUtils.enableSecurityFeatures(installation);
         // Disable security auto-configuration as we want to generate keys/certificates manually here
         ServerUtils.disableSecurityAutoConfiguration(installation);
     }
@@ -100,7 +97,10 @@ public void test40RunWithCert() throws Exception {
         final String certPath = escapePath(installation.config("certs/mynode/mynode.crt"));
         final String caCertPath = escapePath(installation.config("certs/ca/ca.crt"));
 
-        final List<String> tlsConfig = List.of(
+        // Replace possibly auto-configured TLS settings with ones pointing to the material generated with certgen
+        // (we do disable auto-configuration above but for packaged installations TLS auto-config happens on installation time and is
+        // not affected by this setting
+        final List<String> newTlsConfig = List.of(
             "node.name: mynode",
             "xpack.security.transport.ssl.key: " + keyPath,
             "xpack.security.transport.ssl.certificate: " + certPath,
@@ -111,16 +111,16 @@ public void test40RunWithCert() throws Exception {
             "xpack.security.transport.ssl.enabled: true",
             "xpack.security.http.ssl.enabled: true"
         );
-
-        // TODO: Simplify this when https://github.com/elastic/elasticsearch/pull/75144 is merged. We only need to
-        // filter settings from the existing config as they are explicitly set to false on package installation
         List<String> existingConfig = Files.readAllLines(installation.config("elasticsearch.yml"));
         List<String> newConfig = existingConfig.stream()
             .filter(l -> l.startsWith("node.name:") == false)
             .filter(l -> l.startsWith("xpack.security.transport.ssl.") == false)
             .filter(l -> l.startsWith("xpack.security.http.ssl.") == false)
+            .filter(l -> l.startsWith("xpack.security.enabled") == false)
+            .filter(l -> l.startsWith("http.host") == false)
+            .filter(l -> l.startsWith("cluster.initial_master_nodes") == false)
             .collect(Collectors.toList());
-        newConfig.addAll(tlsConfig);
+        newConfig.addAll(newTlsConfig);
 
         Files.write(installation.config("elasticsearch.yml"), newConfig, TRUNCATE_EXISTING);
 

qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java

@@ -38,14 +38,11 @@ public void test20HostnameSubstitution() throws Exception {
             if (distribution.isPackage()) {
                 append(installation.envFile, "HOSTNAME=mytesthost");
             }
-            // Packaged installations don't get autoconfigured yet
-            // TODO: Remove this in https://github.com/elastic/elasticsearch/pull/75144
-            String protocol = distribution.isPackage() ? "http" : "https";
             // security auto-config requires that the archive owner and the node process user be the same
             Platforms.onWindows(() -> sh.chown(confPath, installation.getOwner()));
             assertWhileRunning(() -> {
                 final String nameResponse = ServerUtils.makeRequest(
-                    Request.Get(protocol + "://localhost:9200/_cat/nodes?h=name"),
+                    Request.Get("https://localhost:9200/_cat/nodes?h=name"),
                     "test_superuser",
                     "test_superuser_password",
                     ServerUtils.getCaCert(confPath)

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -84,7 +84,7 @@ public void test11InstallPackageDistribution() throws Exception {
         installation = installPackage(sh, distribution);
         assertInstalled(distribution);
         verifyPackageInstallation(installation, distribution, sh);
-        // We don't add a user here. We explicitly disable security for packages for now after installation. We will update in a followup
+        setFileSuperuser(FILE_REALM_SUPERUSER, FILE_REALM_SUPERUSER_PASSWORD);
 
         final Installation.Executables bin = installation.executables();
         Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool + " has-passwd");
@@ -179,8 +179,7 @@ public void test30KeystorePasswordFromFile() throws Exception {
         assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage());
         Path esKeystorePassphraseFile = installation.config.resolve("eks");
 
-        rmKeystoreIfExists();
-        createKeystore(KEYSTORE_PASSWORD);
+        setKeystorePassword(KEYSTORE_PASSWORD);
 
         assertPasswordProtectedKeystore();
 
@@ -191,7 +190,7 @@ public void test30KeystorePasswordFromFile() throws Exception {
             Files.write(esKeystorePassphraseFile, List.of(KEYSTORE_PASSWORD));
 
             startElasticsearch();
-            ServerUtils.runElasticsearchTests();
+            runElasticsearchTests();
             stopElasticsearch();
         } finally {
             sh.run("sudo systemctl unset-environment ES_KEYSTORE_PASSPHRASE_FILE");

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -59,6 +59,7 @@ public void test10InstallPackage() throws Exception {
         installation = installPackage(sh, distribution());
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), sh);
+        setFileSuperuser("test_superuser", "test_superuser_password");
     }
 
     public void test20PluginsCommandWhenNoPlugins() {
@@ -123,7 +124,7 @@ public void test34CustomJvmOptionsDirectoryFile() throws Exception {
 
         startElasticsearch();
 
-        final String nodesResponse = makeRequest("http://localhost:9200/_nodes");
+        final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
         assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
 
         stopElasticsearch();
@@ -208,18 +209,24 @@ public void test50Remove() throws Exception {
     }
 
     public void test60Reinstall() throws Exception {
-        install();
-        assertInstalled(distribution());
-        verifyPackageInstallation(installation, distribution(), sh);
+        try {
+            install();
+            assertInstalled(distribution());
+            verifyPackageInstallation(installation, distribution(), sh);
 
-        remove(distribution());
-        assertRemoved(distribution());
+            remove(distribution());
+            assertRemoved(distribution());
+        } finally {
+            cleanup();
+        }
     }
 
     public void test70RestartServer() throws Exception {
         try {
             install();
             assertInstalled(distribution());
+            // Recreate file realm users that have been deleted in earlier tests
+            setFileSuperuser("test_superuser", "test_superuser_password");
 
             startElasticsearch();
             restartElasticsearch(sh, installation);
@@ -279,13 +286,15 @@ public void test81CustomPathConfAndJvmOptions() throws Exception {
 
         assertPathsExist(installation.envFile);
         stopElasticsearch();
+        // Recreate file realm users that have been deleted in earlier tests
+        setFileSuperuser("test_superuser", "test_superuser_password");
 
         withCustomConfig(tempConf -> {
             append(installation.envFile, "ES_JAVA_OPTS=\"-Xmx512m -Xms512m -XX:-UseCompressedOops\"");
 
             startElasticsearch();
 
-            final String nodesResponse = makeRequest("http://localhost:9200/_nodes");
+            final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java

@@ -32,6 +32,9 @@ public class PackageUpgradeTests extends PackagingTestCase {
     public void test10InstallBwcVersion() throws Exception {
         installation = installPackage(sh, bwcDistribution);
         assertInstalled(bwcDistribution);
+        // TODO: Add more tests here to assert behavior when updating from < v8 to > v8 with implicit/explicit behavior,
+        // maybe as part of https://github.com/elastic/elasticsearch/pull/76879
+        ServerUtils.disableSecurityFeatures(installation);
     }
 
     public void test11ModifyKeystore() throws Exception {
@@ -78,9 +81,12 @@ public void test20InstallUpgradedVersion() throws Exception {
             installation = Packages.forceUpgradePackage(sh, distribution);
         } else {
             installation = Packages.upgradePackage(sh, distribution);
+            verifySecurityNotAutoConfigured(installation);
         }
         assertInstalled(distribution);
         verifyPackageInstallation(installation, distribution, sh);
+        // Upgrade overwrites the configuration file because we run with --force-confnew so we need to disable security again
+        ServerUtils.disableSecurityFeatures(installation);
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/76283")
@@ -89,28 +95,11 @@ public void test21CheckUpgradedVersion() throws Exception {
     }
 
     private void assertDocsExist() throws Exception {
-        // We can properly handle this as part of https://github.com/elastic/elasticsearch/issues/75940
-        // For now we can use elastic with "keystore.seed" as we set it explicitly in PackageUpgradeTests#test11ModifyKeystore
-        String response1 = ServerUtils.makeRequest(
-            Request.Get("http://localhost:9200/library/_doc/1?pretty"),
-            "elastic",
-            "keystore_seed",
-            null
-        );
+        String response1 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library/_doc/1?pretty"));
         assertThat(response1, containsString("Elasticsearch"));
-        String response2 = ServerUtils.makeRequest(
-            Request.Get("http://localhost:9200/library/_doc/2?pretty"),
-            "elastic",
-            "keystore_seed",
-            null
-        );
+        String response2 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library/_doc/2?pretty"));
         assertThat(response2, containsString("World"));
-        String response3 = ServerUtils.makeRequest(
-            Request.Get("http://localhost:9200/library2/_doc/1?pretty"),
-            "elastic",
-            "keystore_seed",
-            null
-        );
+        String response3 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library2/_doc/1?pretty"));
         assertThat(response3, containsString("Darkness"));
     }
 }