Configure security for the initial node cli

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -66,7 +66,7 @@
 public class DistroTestPlugin implements Plugin<Project> {
     private static final String SYSTEM_JDK_VERSION = "11.0.2+9";
     private static final String SYSTEM_JDK_VENDOR = "openjdk";
-    private static final String GRADLE_JDK_VERSION = "15.0.2+7";
+    private static final String GRADLE_JDK_VERSION = "16.0.1+9";
     private static final String GRADLE_JDK_VENDOR = "adoptopenjdk";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt

@@ -10,7 +10,6 @@ java.nio.file.Path#of(java.lang.String, java.lang.String[]) @ Use org.elasticsea
 java.nio.file.FileSystems#getDefault() @ use org.elasticsearch.core.PathUtils.getDefaultFileSystem() instead.
 
 java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.getFileStore() instead, impacted by JDK-8034057
-java.nio.file.Files#isWritable(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.isWritable() instead, impacted by JDK-8034057
 
 @defaultMessage Use org.elasticsearch.common.Randomness#get for reproducible sources of randomness
 java.util.Random#<init>()

libs/cli/src/main/java/org/elasticsearch/cli/ExitCodes.java

@@ -13,6 +13,7 @@
  */
 public class ExitCodes {
     public static final int OK = 0;
+    public static final int NOOP = 63;           /* nothing to do */
     public static final int USAGE = 64;          /* command line usage error */
     public static final int DATA_ERROR = 65;     /* data format error */
     public static final int NO_INPUT = 66;       /* cannot open input */

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -383,12 +383,6 @@ private void createKeystore(String password) throws Exception {
         final Installation.Executables bin = installation.executables();
         bin.keystoreTool.run("create");
 
-        // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator.
-        // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here.
-        // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests.
-        // when we run these commands as a role user we won't have to do this
-        Platforms.onWindows(() -> sh.chown(keystore));
-
         if (distribution().isDocker()) {
             try {
                 waitForPathToExist(keystore);
@@ -400,6 +394,12 @@ private void createKeystore(String password) throws Exception {
         if (password != null) {
             setKeystorePassword(password);
         }
+
+        // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator.
+        // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here.
+        // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests.
+        // when we run these commands as a role user we won't have to do this
+        Platforms.onWindows(() -> sh.chown(keystore));
     }
 
     private void rmKeystoreIfExists() {

qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java

@@ -197,6 +197,7 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist
             "elasticsearch-certutil",
             "elasticsearch-croneval",
             "elasticsearch-saml-metadata",
+            "elasticsearch-security-config",
             "elasticsearch-setup-passwords",
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -496,6 +496,7 @@ private static void verifyDefaultInstallation(Installation es) {
             "elasticsearch-certutil",
             "elasticsearch-croneval",
             "elasticsearch-saml-metadata",
+            "elasticsearch-security-config",
             "elasticsearch-setup-passwords",
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -185,6 +185,7 @@ public class Executables {
         public final Executable cronevalTool = new Executable("elasticsearch-croneval");
         public final Executable shardTool = new Executable("elasticsearch-shard");
         public final Executable nodeTool = new Executable("elasticsearch-node");
+        public final Executable securityConfigTool = new Executable("elasticsearch-security-config");
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");

server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java

@@ -26,7 +26,6 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
-import java.security.AccessControlException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,11 +34,11 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.containsString;
 
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class ReloadSecureSettingsIT extends ESIntegTestCase {
@@ -399,17 +398,7 @@ public void onFailure(Exception e) {
 
     private SecureSettings writeEmptyKeystore(Environment environment, char[] password) throws Exception {
         final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create();
-        try {
-            keyStoreWrapper.save(environment.configFile(), password);
-        } catch (final AccessControlException e) {
-            if (e.getPermission() instanceof RuntimePermission && e.getPermission().getName().equals("accessUserInformation")) {
-                // this is expected: the save method is extra diligent and wants to make sure
-                // the keystore is readable, not relying on umask and whatnot. It's ok, we don't
-                // care about this in tests.
-            } else {
-                throw e;
-            }
-        }
+        keyStoreWrapper.save(environment.configFile(), password, false);
         return keyStoreWrapper;
     }
 

server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -18,7 +18,6 @@
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.KeyStoreAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.PidFile;
@@ -238,37 +237,11 @@ static SecureSettings loadSecureSettings(Environment initialEnv) throws Bootstra
     }
 
     static SecureSettings loadSecureSettings(Environment initialEnv, InputStream stdin) throws BootstrapException {
-        final KeyStoreWrapper keystore;
         try {
-            keystore = KeyStoreWrapper.load(initialEnv.configFile());
-        } catch (IOException e) {
-            throw new BootstrapException(e);
-        }
-
-        SecureString password;
-        try {
-            if (keystore != null && keystore.hasPassword()) {
-                password = readPassphrase(stdin, KeyStoreAwareCommand.MAX_PASSPHRASE_LENGTH);
-            } else {
-                password = new SecureString(new char[0]);
-            }
-        } catch (IOException e) {
-            throw new BootstrapException(e);
-        }
-
-        try (password) {
-            if (keystore == null) {
-                final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create();
-                keyStoreWrapper.save(initialEnv.configFile(), new char[0]);
-                return keyStoreWrapper;
-            } else {
-                keystore.decrypt(password.getChars());
-                KeyStoreWrapper.upgrade(keystore, initialEnv.configFile(), password.getChars());
-            }
+            return KeyStoreWrapper.bootstrap(initialEnv.configFile(), () -> readPassphrase(stdin, KeyStoreWrapper.MAX_PASSPHRASE_LENGTH));
         } catch (Exception e) {
             throw new BootstrapException(e);
         }
-        return keystore;
     }
 
     // visible for tests

server/src/main/java/org/elasticsearch/cli/KeyStoreAwareCommand.java

@@ -26,9 +26,6 @@ public KeyStoreAwareCommand(String description) {
         super(description);
     }
 
-    /** Arbitrarily chosen maximum passphrase length */
-    public static final int MAX_PASSPHRASE_LENGTH = 128;
-
     /**
      * Reads the keystore password from the {@link Terminal}, prompting for verification where applicable and returns it as a
      * {@link SecureString}.
@@ -42,9 +39,9 @@ protected static SecureString readPassword(Terminal terminal, boolean withVerifi
         final char[] passwordArray;
         if (withVerification) {
             passwordArray = terminal.readSecret("Enter new password for the elasticsearch keystore (empty for no password): ",
-                MAX_PASSPHRASE_LENGTH);
+                KeyStoreWrapper.MAX_PASSPHRASE_LENGTH);
             char[] passwordVerification = terminal.readSecret("Enter same password again: ",
-                MAX_PASSPHRASE_LENGTH);
+                KeyStoreWrapper.MAX_PASSPHRASE_LENGTH);
             if (Arrays.equals(passwordArray, passwordVerification) == false) {
                 throw new UserException(ExitCodes.DATA_ERROR, "Passwords are not equal, exiting.");
             }

server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java

@@ -21,6 +21,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.hash.MessageDigests;
 
@@ -45,6 +46,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.PosixFileAttributeView;
@@ -72,6 +74,9 @@
  */
 public class KeyStoreWrapper implements SecureSettings {
 
+    /** Arbitrarily chosen maximum passphrase length */
+    public static final int MAX_PASSPHRASE_LENGTH = 128;
+
     /** An identifier for the type of data that may be stored in a keystore entry. */
     private enum EntryType {
         STRING,
@@ -101,7 +106,7 @@ private static class Entry {
         "~!@#$%^&*-_=+?").toCharArray();
 
     /** The name of the keystore file to read and write. */
-    private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
+    public static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
 
     /** The version of the metadata written before the keystore data. */
     static final int FORMAT_VERSION = 4;
@@ -194,6 +199,29 @@ public static void addBootstrapSeed(KeyStoreWrapper wrapper) {
         Arrays.fill(characters, (char)0);
     }
 
+    public static KeyStoreWrapper bootstrap(Path configDir, CheckedSupplier<SecureString, Exception> passwordSupplier) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
+
+        SecureString password;
+        if (keystore != null && keystore.hasPassword()) {
+            password = passwordSupplier.get();
+        } else {
+            password = new SecureString(new char[0]);
+        }
+
+        try (password) {
+            if (keystore == null) {
+                final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create();
+                keyStoreWrapper.save(configDir, new char[0]);
+                return keyStoreWrapper;
+            } else {
+                keystore.decrypt(password.getChars());
+                KeyStoreWrapper.upgrade(keystore, configDir, password.getChars());
+            }
+        }
+        return keystore;
+    }
+
     /**
      * Loads information about the Elasticsearch keystore from the provided config directory.
      *
@@ -477,11 +505,16 @@ private void decryptLegacyEntries() throws GeneralSecurityException, IOException
 
     /** Write the keystore to the given config directory. */
     public synchronized void save(Path configDir, char[] password) throws Exception {
+        save(configDir, password, true);
+    }
+
+    public synchronized void save(Path configDir, char[] password, boolean preservePermissions) throws Exception {
         ensureOpen();
 
         Directory directory = new NIOFSDirectory(configDir);
         // write to tmp file first, then overwrite
         String tmpFile = KEYSTORE_FILENAME + ".tmp";
+        Path keystoreTempFile = configDir.resolve(tmpFile);
         try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {
             CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION);
             output.writeByte(password.length == 0 ? (byte)0 : (byte)1);
@@ -515,17 +548,55 @@ public synchronized void save(Path configDir, char[] password) throws Exception
             final String message = String.format(
                 Locale.ROOT,
                 "unable to create temporary keystore at [%s], write permissions required for [%s] or run [elasticsearch-keystore upgrade]",
-                configDir.resolve(tmpFile),
+                keystoreTempFile,
                 configDir);
             throw new UserException(ExitCodes.CONFIG, message, e);
+        } catch (final Exception e) {
+            try {
+                Files.deleteIfExists(keystoreTempFile);
+            } catch (Exception ex) {
+                e.addSuppressed(e);
+            }
+            throw e;
         }
 
         Path keystoreFile = keystorePath(configDir);
-        Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
-        PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class);
-        if (attrs != null) {
-            // don't rely on umask: ensure the keystore has minimal permissions
-            attrs.setPermissions(PosixFilePermissions.fromString("rw-rw----"));
+        if (preservePermissions) {
+            try {
+                // check that replace doesn't change the owner
+                if (Files.exists(keystoreFile, LinkOption.NOFOLLOW_LINKS) &&
+                        false == Files.getOwner(keystoreTempFile, LinkOption.NOFOLLOW_LINKS).equals(Files.getOwner(keystoreFile,
+                                LinkOption.NOFOLLOW_LINKS))) {
+                    String message = String.format(
+                            Locale.ROOT,
+                            "will not overwrite keystore at [%s], because this incurs changing the file owner",
+                            keystoreFile);
+                    throw new UserException(ExitCodes.CONFIG, message);
+                }
+                PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreTempFile, PosixFileAttributeView.class);
+                if (attrs != null) {
+                    // don't rely on umask: ensure the keystore has minimal permissions
+                    attrs.setPermissions(PosixFilePermissions.fromString("rw-rw----"));
+                }
+            } catch (Exception e) {
+                try {
+                    Files.deleteIfExists(keystoreTempFile);
+                } catch (Exception ex) {
+                    e.addSuppressed(ex);
+                }
+                throw e;
+            }
+        }
+
+        try {
+            Files.move(keystoreTempFile, keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+        } catch (Exception e) {
+            try {
+                Files.deleteIfExists(keystoreTempFile);
+            } catch (Exception ex) {
+                e.addSuppressed(ex);
+            }
+            throw e;
         }
     }
 
@@ -584,7 +655,7 @@ public static void validateSettingName(String setting) {
     /**
      * Set a string setting.
      */
-    synchronized void setString(String setting, char[] value) {
+    public synchronized void setString(String setting, char[] value) {
         ensureOpen();
         validateSettingName(setting);
 

x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertGenUtils.java

@@ -31,6 +31,7 @@
 import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
 import org.bouncycastle.pkcs.PKCS10CertificationRequest;
 import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkUtils;
@@ -256,7 +257,7 @@ static PKCS10CertificationRequest generateCSR(KeyPair keyPair, X500Principal pri
      * Gets a random serial for a certificate that is generated from a {@link SecureRandom}
      */
     public static BigInteger getSerial() {
-        SecureRandom random = new SecureRandom();
+        SecureRandom random = Randomness.createSecure();
         BigInteger serial = new BigInteger(SERIAL_BIT_LENGTH, random);
         assert serial.compareTo(BigInteger.valueOf(0L)) >= 0;
         return serial;
@@ -268,7 +269,7 @@ public static BigInteger getSerial() {
     public static KeyPair generateKeyPair(int keysize) throws NoSuchAlgorithmException {
         // generate a private key
         KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
-        keyPairGenerator.initialize(keysize);
+        keyPairGenerator.initialize(keysize, Randomness.createSecure());
         return keyPairGenerator.generateKeyPair();
     }
 
@@ -314,4 +315,13 @@ public static GeneralName createCommonName(String cn) {
         final ASN1Encodable[] sequence = {new ASN1ObjectIdentifier(CN_OID), new DERTaggedObject(true, 0, new DERUTF8String(cn))};
         return new GeneralName(GeneralName.otherName, new DERSequence(sequence));
     }
+
+    /**
+     * See RFC 2247 Using Domains in LDAP/X.500 Distinguished Names
+     * @param domain active directory domain name
+     * @return LDAP DN, distinguished name, of the root of the domain
+     */
+    public static String buildDnFromDomain(String domain) {
+        return "DC=" + domain.replace(".", ",DC=");
+    }
 }