HADOOP-19152. Do not hard code security providers.

hadoop-common-project/hadoop-common/pom.xml

@@ -375,6 +375,7 @@
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk18on</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.kerby</groupId>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JceCtrCryptoCodec.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.crypto;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -27,7 +26,6 @@
 import java.nio.ByteBuffer;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
-import java.security.Security;
 import javax.crypto.Cipher;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
@@ -83,9 +81,6 @@ public Configuration getConf() {
   public void setConf(Configuration conf) {
     this.conf = conf;
     setProvider(conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY));
-    if (BouncyCastleProvider.PROVIDER_NAME.equals(provider)) {
-      Security.addProvider(new BouncyCastleProvider());
-    }
     final String secureRandomAlg =
           conf.get(
               HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java

@@ -26,7 +26,6 @@
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 import java.security.NoSuchAlgorithmException;
-import java.security.Security;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -35,7 +34,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -411,9 +409,6 @@ public KeyProvider(Configuration conf) {
       System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter);
     }
     String jceProvider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
-    if (BouncyCastleProvider.PROVIDER_NAME.equals(jceProvider)) {
-      Security.addProvider(new BouncyCastleProvider());
-    }
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -3623,6 +3623,9 @@ The switch to turn S3A auditing on or off.
   <value></value>
   <description>
     The JCE provider name used in CryptoCodec.
+    If this value is set, the corresponding provider must be added to the provider list.
+    The provider may be added statically in the java.security file, or
+    added dynamically by calling the java.security.Security.addProvider(..) method.
   </description>
 </property>
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java

@@ -32,6 +32,7 @@
 import java.math.BigInteger;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
+import java.security.Security;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -104,6 +105,11 @@ public void testJceAesCtrCryptoCodec() throws Exception {
         jceAesCodecClass, opensslAesCodecClass, iv);
   }
 
+  static void initBouncyCastleProvider(Configuration conf) {
+    conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY, BouncyCastleProvider.PROVIDER_NAME);
+    Security.addProvider(new BouncyCastleProvider());
+  }
+
   @Test(timeout=120000)
   public void testJceSm4CtrCryptoCodec() throws Exception {
     GenericTestUtils.assumeInNativeProfile();
@@ -114,8 +120,7 @@ public void testJceSm4CtrCryptoCodec() throws Exception {
     conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding");
     conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY,
         JceSm4CtrCryptoCodec.class.getName());
-    conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY,
-            BouncyCastleProvider.PROVIDER_NAME);
+    initBouncyCastleProvider(conf);
     Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
     cryptoCodecTest(conf, seed, 0,
         jceSm4CodecClass, jceSm4CodecClass, iv);
@@ -164,8 +169,7 @@ public void testOpensslSm4CtrCryptoCodec() throws Exception {
       LOG.warn("Skipping test since openSSL library not loaded");
       Assume.assumeTrue(false);
     }
-    conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY,
-            BouncyCastleProvider.PROVIDER_NAME);
+    initBouncyCastleProvider(conf);
     Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
     cryptoCodecTest(conf, seed, 0,
         opensslSm4CodecClass, opensslSm4CodecClass, iv);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithJceSm4CtrCryptoCodec.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.crypto;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.junit.BeforeClass;
@@ -26,7 +25,6 @@
     HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
 import static org.assertj.core.api.Assertions.assertThat;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
 
 public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends
     TestCryptoStreams {
@@ -35,8 +33,7 @@ public class TestCryptoStreamsWithJceSm4CtrCryptoCodec extends
   public static void init() throws Exception {
     Configuration conf = new Configuration();
     conf.set(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY, "SM4/CTR/NoPadding");
-    conf.set(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY,
-            BouncyCastleProvider.PROVIDER_NAME);
+    TestCryptoCodec.initBouncyCastleProvider(conf);
     conf.set(
          CommonConfigurationKeysPublic.
              HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_SM4_CTR_NOPADDING_KEY,