Merge branch 'apache:trunk' into HDFS-17769

Author: gyz-web

BUILDING.txt

@@ -29,7 +29,11 @@ This requires a recent version of docker (1.4.1 and higher are known to work).
 On Linux / Mac:
     Install Docker and run this command:
 
-    $ ./start-build-env.sh
+    $ ./start-build-env.sh [OS platform]
+
+   - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
+        Default is 'ubuntu_20'.
+        Note: Currently only default ('ubuntu_20') is supported on arm machine
 
 The prompt which is then presented is located at a mounted version of the source tree
 and all required tools for testing and building have been installed and configured.
@@ -593,18 +597,16 @@ configure the bit-ness of the build, and set several optional components.
 Several tests require that the user must have the Create Symbolic Links
 privilege.
 
-To simplify the installation of Boost, Protocol buffers, OpenSSL and Zlib dependencies we can use
-vcpkg (https://github.com/Microsoft/vcpkg.git). Upon cloning the vcpkg repo, checkout the commit
-7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d to get the required versions of the dependencies
-mentioned above.
-> git clone https://github.com/Microsoft/vcpkg.git
+We use vcpkg (https://github.com/microsoft/vcpkg.git) for installing Boost, Protocol buffers,
+OpenSSL and Zlib dependencies. Run the following commands to setup these dependencies.
+> git clone https://github.com/microsoft/vcpkg.git
 > cd vcpkg
-> git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d
+> git fetch --all
+> git checkout 2025.03.19
 > .\bootstrap-vcpkg.bat
-> .\vcpkg.exe install boost:x64-windows
-> .\vcpkg.exe install protobuf:x64-windows
-> .\vcpkg.exe install openssl:x64-windows
-> .\vcpkg.exe install zlib:x64-windows
+(Assuming that vcpkg was checked out at C:\vcpkg and Hadoop at C:\hadoop)
+> copy C:\hadoop\dev-support\docker\vcpkg\vcpkg.json C:\vcpkg
+> .\vcpkg.exe install --x-install-root .\installed
 
 Set the following environment variables -
 (Assuming that vcpkg was checked out at C:\vcpkg)

LICENSE-binary

@@ -218,12 +218,12 @@ com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.12.7
-com.fasterxml.jackson.core:jackson-core:2.12.7
-com.fasterxml.jackson.core:jackson-databind:2.12.7.1
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
+com.fasterxml.jackson.core:jackson-annotations:2.14.3
+com.fasterxml.jackson.core:jackson-core:2.14.3
+com.fasterxml.jackson.core:jackson-databind:2.14.3
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.3
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.3
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.3
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
@@ -298,12 +298,11 @@ javax.inject:javax.inject:1
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.11.4
-org.apache.avro:avro:1.11.3
 org.apache.commons:commons-compress:1.26.1
 org.apache.commons:commons-configuration2:2.10.1
 org.apache.commons:commons-csv:1.9.0
 org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.12.0
+org.apache.commons:commons-lang3:3.17.0
 org.apache.commons:commons-math3:3.6.1
 org.apache.commons:commons-text:1.10.0
 org.apache.commons:commons-validator:1.6
@@ -505,8 +504,7 @@ javax.cache:cache-api:1.1.1
 javax.servlet:javax.servlet-api:3.1.0
 javax.servlet.jsp:jsp-api:2.1
 javax.websocket:javax.websocket-api:1.0
-javax.ws.rs:jsr311-api:1.1.1
-javax.xml.bind:jaxb-api:2.2.11
+javax.xml.bind:jaxb-api:2.3.1
 
 Eclipse Distribution License (EDL) 1.0
 --------------------------
@@ -538,7 +536,7 @@ org.glassfish.jersey.core:jersey-server:2.46
 org.glassfish.jersey.inject:jersey-hk2:2.46
 org.glassfish.jersey.core:jersey-client:2.46
 org.glassfish.jersey.test-framework:jersey-test-framework-core:2.46
-org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2:2.46
+org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-jetty:2.46
 org.glassfish.jersey.containers:jersey-container-servlet:2.46
 org.glassfish.jersey.containers:jersey-container-servlet-core:2.46
 org.glassfish.jersey.media:jersey-media-json-jettison:2.46

dev-support/docker/Dockerfile_aarch64

@@ -46,6 +46,7 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
 RUN apt-get -q update \
     && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \
     && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends phantomjs \
     && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
     && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc  \
     && apt-get -q update \

dev-support/docker/Dockerfile_ubuntu_24

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM ubuntu:noble
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
+    && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc  \
+    && apt-get -q update \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:noble) \
+    && apt-get clean \
+    && update-java-alternatives -s java-1.17.0-openjdk-amd64 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN locale-gen en_US.UTF-8
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+ENV PYTHONIOENCODING=utf-8
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /usr
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
+
+#######
+# Set env vars for SpotBugs 4.2.2
+#######
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf 3.21.12
+#######
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx3072m
+
+# Skip gpg verification when downloading Yetus via yetus-wrapper
+ENV HADOOP_SKIP_YETUS_VERIFICATION true
+
+####
+# Install packages
+####
+RUN pkg-resolver/install-spotbugs.sh ubuntu:noble
+RUN pkg-resolver/install-boost.sh ubuntu:noble
+RUN pkg-resolver/install-protobuf.sh ubuntu:noble
+RUN pkg-resolver/install-hadolint.sh ubuntu:noble
+RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

dev-support/docker/Dockerfile_windows_10

@@ -49,27 +49,20 @@ RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\G
 # hadolint ignore=DL3003
 RUN powershell git clone https://github.com/microsoft/vcpkg.git \
     && cd vcpkg \
-    && git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
+    && git fetch --all \
+    && git checkout 2025.03.19 \
     && .\bootstrap-vcpkg.bat
 
-# Fix wrong download link for 7-zip in .\vcpkg\scripts\vcpkgTools.xml:
-# - https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0 (not working anymore) is replaced with
-# - https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0 (working)
-# Replacing is done using the Linux tool "sed".
-RUN choco install sed -y
-RUN sed -i "s,https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0,https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0,g" .\vcpkg\scripts\vcpkgTools.xml
+ADD vcpkg/vcpkg.json .
 
-RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
+RUN powershell .\vcpkg\vcpkg.exe install --x-install-root .\vcpkg\installed
 
 # Install Azul Java 8 JDK.
 RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
 
 # Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
+RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
 RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven"
 
 # Install CMake 3.19.0.
@@ -82,16 +75,16 @@ RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -Destination
 RUN setx PATH "%PATH%;C:\ZStd"
 
 # Install libopenssl 3.1.1 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.1-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.1-1-x86_64.pkg.tar
+RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.2-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.2-1-x86_64.pkg.tar.zst
+RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.2-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.1-1-x86_64.pkg.tar
 RUN powershell mkdir "C:\LibOpenSSL"
 RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.1-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
 
-# Install libxxhash 0.8.1 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
+# Install libxxhash 0.8.3 needed for rsync 3.2.7.
+RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.3-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst
+RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar
 RUN powershell mkdir "C:\LibXXHash"
-RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
+RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar -C "C:\LibXXHash"
 
 # Install libzstd 1.5.4 needed for rsync 3.2.7.
 RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.5-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.5-1-x86_64.pkg.tar.zst
@@ -126,6 +119,11 @@ RUN powershell pkg-resolver\install-python.ps1
 RUN powershell pkg-resolver\install-pip.ps1
 RUN powershell pip install python-dateutil
 
+## Install the Microsoft Visual C++ 2010 Redistributable to link leveldbjni native library
+RUN powershell -Command Invoke-WebRequest -Uri https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe -OutFile vcredist_x64.exe
+RUN powershell -Command Start-Process -FilePath .\vcredist_x64.exe -ArgumentList '/quiet', '/norestart' -Wait
+RUN powershell -Command Remove-Item vcredist_x64.exe
+
 # Create a user HadoopBuilder with basic privileges and use it for building Hadoop on Windows.
 RUN powershell New-LocalUser -Name 'HadoopBuilder' -Description 'User account for building Apache Hadoop' -Password ([securestring]::new()) -AccountNeverExpires -PasswordNeverExpires