From 2b3b23f5a72e187061f6bd229fe25e148e424faf Mon Sep 17 00:00:00 2001 From: Balazs Hajgato Date: Thu, 24 Sep 2020 13:07:55 +0200 Subject: [PATCH 1/2] Hadoop 2 RHEL 8 modifications --- .../h/Hadoop/HADOOP-14597.04.patch | 139 ++++++++++++++++++ .../Hadoop-2.10.0-GCCcore-8.3.0-native.eb | 9 +- .../h/Hadoop/Hadoop-2.10.0_tirpc.patch | 15 ++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch create mode 100644 easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch diff --git a/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch b/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch new file mode 100644 index 00000000000..6f805942ae8 --- /dev/null +++ b/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch @@ -0,0 +1,139 @@ +# see: https://issues.apache.org/jira/browse/HADOOP-14597 +# download: https://issues.apache.org/jira/secure/attachment/12875105/HADOOP-14597.04.patch +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 5cb5bba9ae..c7984a3347 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -30,6 +30,11 @@ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); + static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); + static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); + static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); ++static int (*dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); ++static int (*dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++static int (*dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *); ++#endif + static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *, \ + ENGINE *, const unsigned char *, const unsigned char *, int); + static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \ +@@ -46,6 +51,11 @@ typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); + typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *); + typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *); + typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int); ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_test_flags)(const EVP_CIPHER_CTX *, int); ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_block_size)(const EVP_CIPHER_CTX *); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_encrypting)(const EVP_CIPHER_CTX *); ++#endif + typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, \ + const EVP_CIPHER *, ENGINE *, const unsigned char *, \ + const unsigned char *, int); +@@ -60,6 +70,11 @@ static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free; + static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup; + static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init; + static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding; ++static __dlsym_EVP_CIPHER_CTX_test_flags dlsym_EVP_CIPHER_CTX_test_flags; ++static __dlsym_EVP_CIPHER_CTX_block_size dlsym_EVP_CIPHER_CTX_block_size; ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++static __dlsym_EVP_CIPHER_CTX_encrypting dlsym_EVP_CIPHER_CTX_encrypting; ++#endif + static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex; + static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate; + static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex; +@@ -114,6 +129,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + "EVP_CIPHER_CTX_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \ + "EVP_CIPHER_CTX_set_padding"); ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ ++ "EVP_CIPHER_CTX_test_flags"); ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \ ++ "EVP_CIPHER_CTX_block_size"); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \ ++ "EVP_CIPHER_CTX_encrypting"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \ + "EVP_CipherInit_ex"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl, \ +@@ -135,6 +158,17 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding, \ + dlsym_EVP_CIPHER_CTX_set_padding, env, \ + openssl, "EVP_CIPHER_CTX_set_padding"); ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_test_flags, \ ++ dlsym_EVP_CIPHER_CTX_test_flags, env, \ ++ openssl, "EVP_CIPHER_CTX_test_flags"); ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \ ++ dlsym_EVP_CIPHER_CTX_block_size, env, \ ++ openssl, "EVP_CIPHER_CTX_block_size"); ++#if OPENSSL_VERSION_NUMBER >= 0x10100000L ++ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \ ++ dlsym_EVP_CIPHER_CTX_encrypting, env, \ ++ openssl, "EVP_CIPHER_CTX_encrypting"); ++#endif + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \ + env, openssl, "EVP_CipherInit_ex"); + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate, \ +@@ -253,14 +287,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init + static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len, + int max_output_len) + { +- if (context->flags & EVP_CIPH_NO_PADDING) { ++ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) { + if (max_output_len >= input_len) { + return 1; + } + return 0; + } else { +- int b = context->cipher->block_size; ++ int b = dlsym_EVP_CIPHER_CTX_block_size(context); ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + if (context->encrypt) { ++#else ++ if (dlsym_EVP_CIPHER_CTX_encrypting(context)) { ++#endif + if (max_output_len >= input_len + b - 1) { + return 1; + } +@@ -307,10 +345,10 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update + static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context, + int max_output_len) + { +- if (context->flags & EVP_CIPH_NO_PADDING) { ++ if ( dlsym_EVP_CIPHER_CTX_test_flags(context, EVP_CIPH_NO_PADDING) ) { + return 1; + } else { +- int b = context->cipher->block_size; ++ int b = dlsym_EVP_CIPHER_CTX_block_size(context); + if (max_output_len >= b) { + return 1; + } +diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc +index 91fb5a42b2..45cb8c2024 100644 +--- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc ++++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc +@@ -420,6 +420,7 @@ namespace HadoopPipes { + } + + string createDigest(string &password, string& msg) { ++#if OPENSSL_VERSION_NUMBER < 0x10100000L + HMAC_CTX ctx; + unsigned char digest[EVP_MAX_MD_SIZE]; + HMAC_Init(&ctx, (const unsigned char *)password.c_str(), +@@ -428,7 +429,16 @@ namespace HadoopPipes { + unsigned int digestLen; + HMAC_Final(&ctx, digest, &digestLen); + HMAC_cleanup(&ctx); +- ++#else ++ HMAC_CTX *ctx = HMAC_CTX_new(); ++ unsigned char digest[EVP_MAX_MD_SIZE]; ++ HMAC_Init_ex(ctx, (const unsigned char *)password.c_str(), ++ password.length(), EVP_sha1(), NULL); ++ HMAC_Update(ctx, (const unsigned char *)msg.c_str(), msg.length()); ++ unsigned int digestLen; ++ HMAC_Final(ctx, digest, &digestLen); ++ HMAC_CTX_free(ctx); ++#endif + //now apply base64 encoding + BIO *bmem, *b64; + BUF_MEM *bptr; diff --git a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb index d1de46e2bca..0098341f8e6 100644 --- a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb +++ b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb @@ -15,11 +15,15 @@ sources = ['hadoop-%(version)s-src.tar.gz'] patches = [ 'Hadoop-TeraSort-on-local-filesystem.patch', 'Hadoop-2.9.2_fix-zlib.patch', + 'HADOOP-14597.04.patch', + 'Hadoop-2.10.0_tirpc.patch', ] checksums = [ 'baa9b125359a30eb209fbaa953e1b324eb61fa65ceb9a7cf19b0967188d8b1c0', # hadoop-2.10.0-src.tar.gz 'd0a69a6936b4a01505ba2a20911d0cec4f79440dbc8da52b9ddbd7f3a205468b', # Hadoop-TeraSort-on-local-filesystem.patch '1a1d084c7961078bdbaa84716e9639e37587e1d8c0b1f89ce6f12dde8bbbbc5c', # Hadoop-2.9.2_fix-zlib.patch + '68b5b38f6f8482d6fbdcc619c41d99fc64db8d90630b30fdb78b031440c24e72', # HADOOP-14597.04.patch + '85bbf37e7f6d091738774d9a6390c02dc15c8c1f1406cf0dca3ad06165221b1f', # Hadoop-2.10.0_tirpc.patch ] builddependencies = [ @@ -33,7 +37,10 @@ builddependencies = [ ] # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions -dependencies = [('Java', '1.8', '', True)] +dependencies = [ + ('Java', '1.8', '', True), + ('libtirpc', '1.2.6'), +] build_native_libs = True diff --git a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch new file mode 100644 index 00000000000..bcc8c512e21 --- /dev/null +++ b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch @@ -0,0 +1,15 @@ +# RHEL 8 glibc does not includes RPC anymore, so external one has to be used. +# in this case, SerialUtils.cc have to be licked with the external library, libtirpc. +# SEP 24th 2020 by B. Hajgato (UGent) +diff -ru hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt hadoop-2.10.0-src/hadoop-tools/hadoop-pipes/src/CMakeLists.txt +--- hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt 2019-10-15 20:12:38.000000000 +0200 ++++ hadoop-2.10.0-src/hadoop-tools/hadoop-pipes/src/CMakeLists.txt 2020-09-24 12:02:21.022819284 +0200 +@@ -63,6 +63,8 @@ + set(LIB_DL "dl") + endif() + ++set(LIB_NET "tirpc") ++ + if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + exec_program("uname" ARGS "-r" OUTPUT_VARIABLE OS_VERSION) + if(OS_VERSION VERSION_LESS "5.12") From f39f6402aff2451af1b05cf6a931f8becdc5a81f Mon Sep 17 00:00:00 2001 From: Balazs Hajgato Date: Thu, 24 Sep 2020 13:34:28 +0200 Subject: [PATCH 2/2] fix typos --- easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch | 1 + .../h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb | 4 ++-- easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch b/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch index 6f805942ae8..e6503ee7119 100644 --- a/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch +++ b/easybuild/easyconfigs/h/Hadoop/HADOOP-14597.04.patch @@ -1,3 +1,4 @@ +# Patch for newer OpenSSL (1.1.x) # see: https://issues.apache.org/jira/browse/HADOOP-14597 # download: https://issues.apache.org/jira/secure/attachment/12875105/HADOOP-14597.04.patch diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c diff --git a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb index 0098341f8e6..78f007fd28d 100644 --- a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb +++ b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0-GCCcore-8.3.0-native.eb @@ -22,8 +22,8 @@ checksums = [ 'baa9b125359a30eb209fbaa953e1b324eb61fa65ceb9a7cf19b0967188d8b1c0', # hadoop-2.10.0-src.tar.gz 'd0a69a6936b4a01505ba2a20911d0cec4f79440dbc8da52b9ddbd7f3a205468b', # Hadoop-TeraSort-on-local-filesystem.patch '1a1d084c7961078bdbaa84716e9639e37587e1d8c0b1f89ce6f12dde8bbbbc5c', # Hadoop-2.9.2_fix-zlib.patch - '68b5b38f6f8482d6fbdcc619c41d99fc64db8d90630b30fdb78b031440c24e72', # HADOOP-14597.04.patch - '85bbf37e7f6d091738774d9a6390c02dc15c8c1f1406cf0dca3ad06165221b1f', # Hadoop-2.10.0_tirpc.patch + 'ea93c7c2b03d36f1434c2f2921c031cdc385a98f337ed8f4b3103b45b0ad0da3', # HADOOP-14597.04.patch + '9d66f604e6e03923d8fcb290382936fb93511001bb593025b8d63ababdca3a96', # Hadoop-2.10.0_tirpc.patch ] builddependencies = [ diff --git a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch index bcc8c512e21..09834a0126c 100644 --- a/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch +++ b/easybuild/easyconfigs/h/Hadoop/Hadoop-2.10.0_tirpc.patch @@ -1,5 +1,5 @@ -# RHEL 8 glibc does not includes RPC anymore, so external one has to be used. -# in this case, SerialUtils.cc have to be licked with the external library, libtirpc. +# RHEL 8 glibc does not include RPC anymore, so external one has to be used. +# And SerialUtils.cc has to be linked with the external library, libtirpc # SEP 24th 2020 by B. Hajgato (UGent) diff -ru hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt hadoop-2.10.0-src/hadoop-tools/hadoop-pipes/src/CMakeLists.txt --- hadoop-2.10.0-src.orig/hadoop-tools/hadoop-pipes/src/CMakeLists.txt 2019-10-15 20:12:38.000000000 +0200