From 1896095a7ff9393d3547e82b70109469c5b13890 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 27 Jan 2024 22:45:31 +0800 Subject: [PATCH 1/2] HBASE-28331 Client integration test fails after upgrading hadoop3 version to 3.3.x Patch hadoop 3.3.x to make minicluster work again --- dev-support/Jenkinsfile | 2 ++ dev-support/patch-hadoop3.sh | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100755 dev-support/patch-hadoop3.sh diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 9c03d97a3c44..5ff411ee473c 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -726,6 +726,8 @@ pipeline { echo "Attempting to use run an instance on top of Hadoop 3." artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1) tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3" + # we need to patch some files otherwise minicluster will fail to start, see MAPREDUCE-7471 + ${BASEDIR}/dev-support/patch-hadoop3.sh hadoop-3 if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \ --single-process \ --working-dir output-integration/hadoop-3 \ diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh new file mode 100755 index 000000000000..a5302c4c7a18 --- /dev/null +++ b/dev-support/patch-hadoop3.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +hadoop_dir=$1 + +sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" $1/libexec/hadoop-functions.sh +sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" $1/bin/mapred +curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o $1/share/hadoop/tools/lib/mockito-core-2.28.2.jar From d6872aa46ee3a3cf2c75d5316ba05f33195f0e59 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Mon, 29 Jan 2024 23:10:36 +0800 Subject: [PATCH 2/2] fix shell check --- dev-support/patch-hadoop3.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-support/patch-hadoop3.sh b/dev-support/patch-hadoop3.sh index a5302c4c7a18..fed5a535bcdd 100755 --- a/dev-support/patch-hadoop3.sh +++ b/dev-support/patch-hadoop3.sh @@ -19,6 +19,6 @@ hadoop_dir=$1 -sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" $1/libexec/hadoop-functions.sh -sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" $1/bin/mapred -curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o $1/share/hadoop/tools/lib/mockito-core-2.28.2.jar +sed -i "s/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"share\/hadoop\/tools\"}/HADOOP_TOOLS_DIR=\${HADOOP_TOOLS_DIR:-\"\$HADOOP_TOOLS_HOME\/share\/hadoop\/tools\"}/g" "$hadoop_dir/libexec/hadoop-functions.sh" +sed -i "/hadoop_add_classpath \"\${junitjar}\"/a mockitojar=\$(echo \"\${HADOOP_TOOLS_LIB_JARS_DIR}\"\/mockito-core-[0-9]*.jar)\nhadoop_add_classpath \"\${mockitojar}\"" "$hadoop_dir/bin/mapred" +curl https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar -o "$hadoop_dir/share/hadoop/tools/lib/mockito-core-2.28.2.jar"