diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index e41341a70277a..c59f5a5bddeca 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -254,7 +254,7 @@ def commonPackageConfig(String type, String architecture) { } } -// this is package indepdendent configuration +// this is package independent configuration ospackage { maintainer 'Elasticsearch Team ' summary 'Distributed RESTful search engine built for the cloud' diff --git a/distribution/packages/src/common/scripts/postinst b/distribution/packages/src/common/scripts/postinst index e73a095556470..a2b4c2930851f 100644 --- a/distribution/packages/src/common/scripts/postinst +++ b/distribution/packages/src/common/scripts/postinst @@ -49,13 +49,58 @@ case "$1" in exit 1 ;; esac - # to pick up /usr/lib/sysctl.d/elasticsearch.conf if command -v systemctl > /dev/null; then systemctl restart systemd-sysctl.service || true fi - if [ "x$IS_UPGRADE" != "xtrue" ]; then + # Don't exit immediately on error, we want to hopefully print some helpful banners + set +e + # Attempt to auto-configure security, this seems to be an installation + if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \ + ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \ + ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \ + /usr/share/elasticsearch/bin/elasticsearch-cli <<< ""; then + # Above command runs as root and TLS keystores are created group-owned by root. It's simple to correct the ownership here + for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_* + do + chown root:elasticsearch "${dir}"/http_keystore_local_node.p12 + chown root:elasticsearch "${dir}"/http_ca.crt + chown root:elasticsearch "${dir}"/transport_keystore_all_nodes.p12 + done + if INITIAL_PASSWORD=$(ES_MAIN_CLASS=org.elasticsearch.xpack.security.enrollment.tool.AutoConfigGenerateElasticPasswordHash \ + ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \ + ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \ + /usr/share/elasticsearch/bin/elasticsearch-cli); then + echo "########## Security autoconfiguration information ############" + echo "# #" + echo "# Authentication and Authorization are enabled. #" + echo "# TLS for the transport and the http layers is enabled and configured. #" + echo "# #" + echo "# The password of the elastic superuser will be set to: ${INITIAL_PASSWORD} #" + echo "# upon starting elasticsearch for the first time #" + echo "# #" + echo "##############################################################################" + fi + else + if [ $? -eq 80 ]; then + # ExitCodes.NOOP + echo "########## Security autoconfiguration information ############" + echo "# #" + echo "# Security features appear to be already configured. #" + echo "# #" + echo "##############################################################################" + else + echo "########## Security autoconfiguration information ############" + echo "# #" + echo "# Failed to auto-configure security features. #" + echo "# Authentication and Authorization are enabled. #" + echo "# You can use elasticsearch-reset-elastic-password to set a password #" + echo "# for the elastic user. #" + echo "# #" + echo "##############################################################################" + fi + fi if command -v systemctl >/dev/null; then echo "### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd" echo " sudo systemctl daemon-reload" @@ -63,6 +108,8 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then echo "### You can start elasticsearch service by executing" echo " sudo systemctl start elasticsearch.service" fi + set -e + elif [ "$RESTART_ON_UPGRADE" = "true" ]; then echo -n "Restarting elasticsearch service..." diff --git a/distribution/packages/src/common/scripts/postrm b/distribution/packages/src/common/scripts/postrm index 0fc3aca400c1d..cc1032c0e73b7 100644 --- a/distribution/packages/src/common/scripts/postrm +++ b/distribution/packages/src/common/scripts/postrm @@ -18,6 +18,7 @@ export ES_PATH_CONF=${ES_PATH_CONF:-@path.conf@} REMOVE_DIRS=false REMOVE_JVM_OPTIONS_DIRECTORY=false +REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY=false REMOVE_ELASTICSEARCH_KEYSTORE=false REMOVE_USER_AND_GROUP=false @@ -31,6 +32,7 @@ case "$1" in purge) REMOVE_DIRS=true REMOVE_JVM_OPTIONS_DIRECTORY=true + REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY=true REMOVE_ELASTICSEARCH_KEYSTORE=true REMOVE_USER_AND_GROUP=true ;; @@ -99,6 +101,16 @@ if [ "$REMOVE_DIRS" = "true" ]; then fi fi + # delete the security auto config directory if we are purging + if [ "$REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY" = "true" ]; then + for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_* + do + echo -n "Deleting security auto-configuration directory..." + rm -rf "${dir}" + echo "OK" + done + fi + # delete the elasticsearch keystore if we are purging if [ "$REMOVE_ELASTICSEARCH_KEYSTORE" = "true" ]; then if [ -e "${ES_PATH_CONF}/elasticsearch.keystore" ]; then diff --git a/docs/changelog/75144.yaml b/docs/changelog/75144.yaml new file mode 100644 index 0000000000000..ec7558dbe10ad --- /dev/null +++ b/docs/changelog/75144.yaml @@ -0,0 +1,6 @@ +pr: 75144 +summary: Security auto-configuration for packaged installations +area: Security +type: enhancement +issues: + - 78306 diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java index 856f02b5d937c..c5a7694c65d54 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/CertGenCliTests.java @@ -48,9 +48,6 @@ public static void cleanupFiles() { public void test10Install() throws Exception { install(); - // Enable security for this test only where it is necessary, until we can enable it for all - // Only needed until https://github.com/elastic/elasticsearch/pull/75144 is merged - ServerUtils.enableSecurityFeatures(installation); // Disable security auto-configuration as we want to generate keys/certificates manually here ServerUtils.disableSecurityAutoConfiguration(installation); } @@ -100,7 +97,10 @@ public void test40RunWithCert() throws Exception { final String certPath = escapePath(installation.config("certs/mynode/mynode.crt")); final String caCertPath = escapePath(installation.config("certs/ca/ca.crt")); - final List tlsConfig = List.of( + // Replace possibly auto-configured TLS settings with ones pointing to the material generated with certgen + // (we do disable auto-configuration above but for packaged installations TLS auto-config happens on installation time and is + // not affected by this setting + final List newTlsConfig = List.of( "node.name: mynode", "xpack.security.transport.ssl.key: " + keyPath, "xpack.security.transport.ssl.certificate: " + certPath, @@ -111,16 +111,16 @@ public void test40RunWithCert() throws Exception { "xpack.security.transport.ssl.enabled: true", "xpack.security.http.ssl.enabled: true" ); - - // TODO: Simplify this when https://github.com/elastic/elasticsearch/pull/75144 is merged. We only need to - // filter settings from the existing config as they are explicitly set to false on package installation List existingConfig = Files.readAllLines(installation.config("elasticsearch.yml")); List newConfig = existingConfig.stream() .filter(l -> l.startsWith("node.name:") == false) .filter(l -> l.startsWith("xpack.security.transport.ssl.") == false) .filter(l -> l.startsWith("xpack.security.http.ssl.") == false) + .filter(l -> l.startsWith("xpack.security.enabled") == false) + .filter(l -> l.startsWith("http.host") == false) + .filter(l -> l.startsWith("cluster.initial_master_nodes") == false) .collect(Collectors.toList()); - newConfig.addAll(tlsConfig); + newConfig.addAll(newTlsConfig); Files.write(installation.config("elasticsearch.yml"), newConfig, TRUNCATE_EXISTING); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java index 91ba054bf3dcc..6dd68416c4298 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java @@ -38,14 +38,11 @@ public void test20HostnameSubstitution() throws Exception { if (distribution.isPackage()) { append(installation.envFile, "HOSTNAME=mytesthost"); } - // Packaged installations don't get autoconfigured yet - // TODO: Remove this in https://github.com/elastic/elasticsearch/pull/75144 - String protocol = distribution.isPackage() ? "http" : "https"; // security auto-config requires that the archive owner and the node process user be the same Platforms.onWindows(() -> sh.chown(confPath, installation.getOwner())); assertWhileRunning(() -> { final String nameResponse = ServerUtils.makeRequest( - Request.Get(protocol + "://localhost:9200/_cat/nodes?h=name"), + Request.Get("https://localhost:9200/_cat/nodes?h=name"), "test_superuser", "test_superuser_password", ServerUtils.getCaCert(confPath) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index cef55bbcb37d9..bdac50050c5bd 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -84,7 +84,7 @@ public void test11InstallPackageDistribution() throws Exception { installation = installPackage(sh, distribution); assertInstalled(distribution); verifyPackageInstallation(installation, distribution, sh); - // We don't add a user here. We explicitly disable security for packages for now after installation. We will update in a followup + setFileSuperuser(FILE_REALM_SUPERUSER, FILE_REALM_SUPERUSER_PASSWORD); final Installation.Executables bin = installation.executables(); Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool + " has-passwd"); @@ -179,8 +179,7 @@ public void test30KeystorePasswordFromFile() throws Exception { assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage()); Path esKeystorePassphraseFile = installation.config.resolve("eks"); - rmKeystoreIfExists(); - createKeystore(KEYSTORE_PASSWORD); + setKeystorePassword(KEYSTORE_PASSWORD); assertPasswordProtectedKeystore(); @@ -191,7 +190,7 @@ public void test30KeystorePasswordFromFile() throws Exception { Files.write(esKeystorePassphraseFile, List.of(KEYSTORE_PASSWORD)); startElasticsearch(); - ServerUtils.runElasticsearchTests(); + runElasticsearchTests(); stopElasticsearch(); } finally { sh.run("sudo systemctl unset-environment ES_KEYSTORE_PASSPHRASE_FILE"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index bf00d7d6fc184..096e01b1f737f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -59,6 +59,7 @@ public void test10InstallPackage() throws Exception { installation = installPackage(sh, distribution()); assertInstalled(distribution()); verifyPackageInstallation(installation, distribution(), sh); + setFileSuperuser("test_superuser", "test_superuser_password"); } public void test20PluginsCommandWhenNoPlugins() { @@ -123,7 +124,7 @@ public void test34CustomJvmOptionsDirectoryFile() throws Exception { startElasticsearch(); - final String nodesResponse = makeRequest("http://localhost:9200/_nodes"); + final String nodesResponse = makeRequest("https://localhost:9200/_nodes"); assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); stopElasticsearch(); @@ -208,18 +209,24 @@ public void test50Remove() throws Exception { } public void test60Reinstall() throws Exception { - install(); - assertInstalled(distribution()); - verifyPackageInstallation(installation, distribution(), sh); + try { + install(); + assertInstalled(distribution()); + verifyPackageInstallation(installation, distribution(), sh); - remove(distribution()); - assertRemoved(distribution()); + remove(distribution()); + assertRemoved(distribution()); + } finally { + cleanup(); + } } public void test70RestartServer() throws Exception { try { install(); assertInstalled(distribution()); + // Recreate file realm users that have been deleted in earlier tests + setFileSuperuser("test_superuser", "test_superuser_password"); startElasticsearch(); restartElasticsearch(sh, installation); @@ -279,13 +286,15 @@ public void test81CustomPathConfAndJvmOptions() throws Exception { assertPathsExist(installation.envFile); stopElasticsearch(); + // Recreate file realm users that have been deleted in earlier tests + setFileSuperuser("test_superuser", "test_superuser_password"); withCustomConfig(tempConf -> { append(installation.envFile, "ES_JAVA_OPTS=\"-Xmx512m -Xms512m -XX:-UseCompressedOops\""); startElasticsearch(); - final String nodesResponse = makeRequest("http://localhost:9200/_nodes"); + final String nodesResponse = makeRequest("https://localhost:9200/_nodes"); assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java index 9cd74f04c289f..d7f03fb2a0298 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageUpgradeTests.java @@ -32,6 +32,9 @@ public class PackageUpgradeTests extends PackagingTestCase { public void test10InstallBwcVersion() throws Exception { installation = installPackage(sh, bwcDistribution); assertInstalled(bwcDistribution); + // TODO: Add more tests here to assert behavior when updating from < v8 to > v8 with implicit/explicit behavior, + // maybe as part of https://github.com/elastic/elasticsearch/pull/76879 + ServerUtils.disableSecurityFeatures(installation); } public void test11ModifyKeystore() throws Exception { @@ -78,9 +81,12 @@ public void test20InstallUpgradedVersion() throws Exception { installation = Packages.forceUpgradePackage(sh, distribution); } else { installation = Packages.upgradePackage(sh, distribution); + verifySecurityNotAutoConfigured(installation); } assertInstalled(distribution); verifyPackageInstallation(installation, distribution, sh); + // Upgrade overwrites the configuration file because we run with --force-confnew so we need to disable security again + ServerUtils.disableSecurityFeatures(installation); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/76283") @@ -89,28 +95,11 @@ public void test21CheckUpgradedVersion() throws Exception { } private void assertDocsExist() throws Exception { - // We can properly handle this as part of https://github.com/elastic/elasticsearch/issues/75940 - // For now we can use elastic with "keystore.seed" as we set it explicitly in PackageUpgradeTests#test11ModifyKeystore - String response1 = ServerUtils.makeRequest( - Request.Get("http://localhost:9200/library/_doc/1?pretty"), - "elastic", - "keystore_seed", - null - ); + String response1 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library/_doc/1?pretty")); assertThat(response1, containsString("Elasticsearch")); - String response2 = ServerUtils.makeRequest( - Request.Get("http://localhost:9200/library/_doc/2?pretty"), - "elastic", - "keystore_seed", - null - ); + String response2 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library/_doc/2?pretty")); assertThat(response2, containsString("World")); - String response3 = ServerUtils.makeRequest( - Request.Get("http://localhost:9200/library2/_doc/1?pretty"), - "elastic", - "keystore_seed", - null - ); + String response3 = ServerUtils.makeRequest(Request.Get("http://localhost:9200/library2/_doc/1?pretty")); assertThat(response3, containsString("Darkness")); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java new file mode 100644 index 0000000000000..1affcd7646f96 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.packaging.test; + +import org.elasticsearch.packaging.util.Installation; +import org.elasticsearch.packaging.util.Packages; +import org.junit.BeforeClass; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.Packages.assertInstalled; +import static org.elasticsearch.packaging.util.Packages.assertRemoved; +import static org.elasticsearch.packaging.util.Packages.installPackage; +import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.Assume.assumeTrue; + +public class PackagesSecurityAutoConfigurationTests extends PackagingTestCase { + + @BeforeClass + public static void filterDistros() { + assumeTrue("rpm or deb", distribution.isPackage()); + } + + public void test10SecurityAutoConfiguredOnPackageInstall() throws Exception { + assertRemoved(distribution()); + installation = installPackage(sh, distribution(), successfulAutoConfiguration()); + assertInstalled(distribution()); + verifyPackageInstallation(installation, distribution(), sh); + verifySecurityAutoConfigured(installation); + assertNotNull(installation.getElasticPassword()); + } + + public void test20SecurityNotAutoConfiguredOnReInstallation() throws Exception { + // we are testing force upgrading in the current version + // In such a case, security remains configured from the initial installation, we don't run it again. + Optional autoConfigDirName = getAutoConfigDirName(installation); + installation = Packages.forceUpgradePackage(sh, distribution); + assertInstalled(distribution); + verifyPackageInstallation(installation, distribution, sh); + verifySecurityAutoConfigured(installation); + // Since we did not auto-configure the second time, the directory name should be the same + assertThat(autoConfigDirName.isPresent(), is(true)); + assertThat(getAutoConfigDirName(installation).isPresent(), is(true)); + assertThat(getAutoConfigDirName(installation).get(), equalTo(autoConfigDirName.get())); + } + + public void test30SecurityNotAutoConfiguredWhenExistingDataDir() throws Exception { + // This is a contrived example for packages where in a new installation, there is an + // existing data directory but the rest of the package tracked config files were removed + final Path dataPath = installation.data; + cleanup(); + Files.createDirectory(dataPath); + append(dataPath.resolve("foo"), "some data"); + installation = installPackage(sh, distribution(), existingSecurityConfiguration()); + verifySecurityNotAutoConfigured(installation); + } + + public void test40SecurityNotAutoConfiguredWhenExistingKeystoreUnknownPassword() throws Exception { + // This is a contrived example for packages where in a new installation, there is an + // existing elasticsearch.keystore file within $ES_PATH_CONF and it's password-protected + final Installation.Executables bin = installation.executables(); + bin.keystoreTool.run("passwd", "some_password\nsome_password\n"); + final Path tempDir = createTempDir("existing-keystore-config"); + final Path confPath = installation.config; + Files.copy( + confPath.resolve("elasticsearch.keystore"), + tempDir.resolve("elasticsearch.keystore"), + StandardCopyOption.COPY_ATTRIBUTES + ); + cleanup(); + Files.createDirectory(confPath); + Files.copy( + tempDir.resolve("elasticsearch.keystore"), + confPath.resolve("elasticsearch.keystore"), + StandardCopyOption.COPY_ATTRIBUTES + ); + installation = installPackage(sh, distribution(), errorOutput()); + List configLines = Files.readAllLines(installation.config("elasticsearch.yml")); + assertThat(configLines, not(hasItem("# have been automatically generated in order to configure Security. #"))); + } + + private Predicate successfulAutoConfiguration() { + Predicate p1 = output -> output.contains("Authentication and Authorization are enabled."); + Predicate p2 = output -> output.contains("TLS for the transport and the http layers is enabled and configured."); + Predicate p3 = output -> output.contains("The password of the elastic superuser will be set to:"); + return p1.and(p2).and(p3); + } + + private Predicate existingSecurityConfiguration() { + return output -> output.contains("Security features appear to be already configured."); + } + + private Predicate errorOutput() { + Predicate p1 = output -> output.contains("Failed to auto-configure security features."); + Predicate p2 = output -> output.contains("Authentication and Authorization are enabled."); + Predicate p3 = output -> output.contains("You can use elasticsearch-reset-elastic-password to set a password"); + Predicate p4 = output -> output.contains("for the elastic user."); + return p1.and(p2).and(p3).and(p4); + } + +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 104c12228da0f..c3c228ae7ec2b 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -671,7 +671,7 @@ public void verifySecurityAutoConfigured(Installation es) throws Exception { Stream.of("http_keystore_local_node.p12", "http_ca.crt", "transport_keystore_all_nodes.p12") .forEach(file -> assertThat(es.config(autoConfigDirName.get()).resolve(file), FileMatcher.file(File, owner, owner, p660))); configLines = Files.readAllLines(es.config("elasticsearch.yml")); - } else { + } else if (es.distribution.isDocker()) { assertThat(es.config(autoConfigDirName.get()), DockerFileMatcher.file(Directory, "elasticsearch", "root", p750)); Stream.of("http_keystore_local_node.p12", "http_ca.crt", "transport_keystore_all_nodes.p12") .forEach( @@ -685,8 +685,19 @@ public void verifySecurityAutoConfigured(Installation es) throws Exception { configLines = Files.readAllLines(localTempDir.resolve("docker_elasticsearch.yml")); rm(localTempDir.resolve("docker_elasticsearch.yml")); rm(localTempDir); + } else { + assert es.distribution.isPackage(); + assertThat(es.config(autoConfigDirName.get()), FileMatcher.file(Directory, "root", "elasticsearch", p750)); + Stream.of("http_keystore_local_node.p12", "http_ca.crt", "transport_keystore_all_nodes.p12") + .forEach( + file -> assertThat( + es.config(autoConfigDirName.get()).resolve(file), + FileMatcher.file(File, "root", "elasticsearch", p660) + ) + ); + assertThat(sh.run(es.executables().keystoreTool + " list").stdout, Matchers.containsString("autoconfiguration.password_hash")); + configLines = Files.readAllLines(es.config("elasticsearch.yml")); } - assertThat(configLines, hasItem("xpack.security.enabled: true")); assertThat(configLines, hasItem("xpack.security.http.ssl.enabled: true")); assertThat(configLines, hasItem("xpack.security.transport.ssl.enabled: true")); @@ -723,6 +734,12 @@ public void verifySecurityAutoConfigured(Installation es) throws Exception { */ public static void verifySecurityNotAutoConfigured(Installation es) throws Exception { assertThat(getAutoConfigDirName(es).isPresent(), Matchers.is(false)); + if (es.distribution.isPackage()) { + assertThat( + sh.run(es.executables().keystoreTool + " list").stdout, + not(Matchers.containsString("autoconfiguration.password_hash")) + ); + } List configLines = Files.readAllLines(es.config("elasticsearch.yml")); assertThat(configLines, not(contains(containsString("automatically generated in order to configure Security")))); Path caCert = ServerUtils.getCaCert(installation); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java index 0dc1dc3eba454..e52f0146ecc8f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java @@ -34,14 +34,12 @@ public class PasswordToolsTests extends PackagingTestCase { @Before public void filterDistros() { - assumeFalse("no docker", distribution.isDocker()); + assumeFalse("only archives", distribution.isDocker() || distribution().isPackage()); } public void test010Install() throws Exception { install(); - // Enable security for this test only where it is necessary, until we can enable it for all - ServerUtils.enableSecurityFeatures(installation); - // Disable auto-configuration so that we can run setup-passwords + // Disable auto-configuration for archives so that we can run setup-passwords ServerUtils.disableSecurityAutoConfiguration(installation); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java index 0195af1d54c38..28eacfe6c2645 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PluginCliTests.java @@ -28,6 +28,7 @@ public class PluginCliTests extends PackagingTestCase { private static final String EXAMPLE_PLUGIN_NAME = "custom-settings"; private static final Path EXAMPLE_PLUGIN_ZIP; + static { // re-read before each test so the plugin path can be manipulated within tests EXAMPLE_PLUGIN_ZIP = Paths.get(System.getProperty("tests.example-plugin", "/dummy/path")); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index 3374184f883bb..ee6b4995a218d 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -8,6 +8,8 @@ package org.elasticsearch.packaging.util; +import org.elasticsearch.core.Nullable; + import java.nio.file.Path; import java.nio.file.Paths; @@ -32,6 +34,8 @@ public class Installation { public final Path modules; public final Path pidDir; public final Path envFile; + @Nullable + private String elasticPassword; // auto-configured password upon installation private Installation( Shell sh, @@ -58,6 +62,7 @@ private Installation( this.modules = modules; this.pidDir = pidDir; this.envFile = envFile; + this.elasticPassword = null; } public static Installation ofArchive(Shell sh, Distribution distribution, Path home) { @@ -136,6 +141,14 @@ public Path config(Path configFileName) { return config.resolve(configFileName); } + public String getElasticPassword() { + return this.elasticPassword; + } + + public void setElasticPassword(String password) { + this.elasticPassword = password; + } + public Executables executables() { return new Executables(); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index 5153dc8bde0c5..39bea196c9d8a 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; +import org.elasticsearch.core.Nullable; import org.elasticsearch.packaging.util.Shell.Result; import java.io.IOException; @@ -18,8 +19,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -75,7 +78,12 @@ public static Result packageStatus(Distribution distribution) { return runPackageManager(distribution, new Shell(), PackageManagerCommand.QUERY); } - public static Installation installPackage(Shell sh, Distribution distribution) throws IOException { + public static Installation installPackage(Shell sh, Distribution distribution) throws Exception { + return installPackage(sh, distribution, null); + } + + public static Installation installPackage(Shell sh, Distribution distribution, @Nullable Predicate outputPredicate) + throws IOException { String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); if (distribution.hasJdk == false) { sh.getEnv().put("ES_JAVA_HOME", systemJavaHome); @@ -84,9 +92,11 @@ public static Installation installPackage(Shell sh, Distribution distribution) t if (result.exitCode != 0) { throw new RuntimeException("Installing distribution " + distribution + " failed: " + result); } - + if (null != outputPredicate) { + assertThat(outputPredicate.test(result.stdout), is(true)); + } Installation installation = Installation.ofPackage(sh, distribution); - + installation.setElasticPassword(captureElasticPasswordFromOutput(result)); if (distribution.hasJdk == false) { Files.write(installation.envFile, List.of("ES_JAVA_HOME=" + systemJavaHome), StandardOpenOption.APPEND); } @@ -94,12 +104,18 @@ public static Installation installPackage(Shell sh, Distribution distribution) t if (Version.fromString(distribution.baseVersion).onOrAfter(Version.V_7_13_0)) { ServerUtils.disableGeoIpDownloader(installation); } - // https://github.com/elastic/elasticsearch/issues/75940 - // TODO Figure out how to run all packaging tests with security enabled which is now the default behavior - ServerUtils.disableSecurityFeatures(installation); + return installation; } + private static String captureElasticPasswordFromOutput(Result result) { + return Arrays.stream(result.stdout.split(System.lineSeparator())) + .filter(l -> l.contains("The password of the elastic superuser will be set to:")) + .map(l -> l.substring(56, 76)) + .findFirst() + .orElse(null); + } + public static Installation upgradePackage(Shell sh, Distribution distribution) throws IOException { final Result result = runPackageManager(distribution, sh, PackageManagerCommand.UPGRADE); if (result.exitCode != 0) { @@ -156,7 +172,7 @@ public static void remove(Distribution distribution) throws Exception { }); } - public static void verifyPackageInstallation(Installation installation, Distribution distribution, Shell sh) { + public static void verifyPackageInstallation(Installation installation, Distribution distribution, Shell sh) throws IOException { verifyOssInstallation(installation, distribution, sh); verifyDefaultInstallation(installation, distribution); } diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java index e0036b721cbfc..cf3d101700165 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java @@ -45,7 +45,6 @@ import org.elasticsearch.xpack.core.ssl.CertParsingUtils; import javax.security.auth.x500.X500Principal; - import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter;