From b89aaa825c18bb11eb799dd020972083ec6a7c20 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Fri, 9 May 2025 18:49:19 +0530 Subject: [PATCH 01/13] HBASE-29225 Add module for Jetty 12 with EE8 to hbase-thirdparty (#131) This change adds a new set of modules which we will need for Jetty 12 migration. The code has been modularised into following modules: - `hbase-shaded-jetty-12-plus-core`: Contains shaded jetty 12 core jars - `hbase-shaded-jetty-12-plus-ee8`: Contains shaded jetty EE8 specific jars So basically: - Branches which want to consume EE8 may need to add `hbase-shaded-jetty-12-plus-core` and `hbase-shaded-jetty-12-plus-ee8` in their dependency replacing the former `hbase-shaded-jetty` - Branches which want to consume EE9/EE10 may need to add a new module for same in `hbase-thirdparty` in future Signed-off-by: Istvan Toth --- hbase-shaded-jetty-12-plus-core/pom.xml | 223 +++++++++++++++++++++ hbase-shaded-jetty-12-plus-ee8/pom.xml | 252 ++++++++++++++++++++++++ pom.xml | 3 + 3 files changed, 478 insertions(+) create mode 100644 hbase-shaded-jetty-12-plus-core/pom.xml create mode 100644 hbase-shaded-jetty-12-plus-ee8/pom.xml diff --git a/hbase-shaded-jetty-12-plus-core/pom.xml b/hbase-shaded-jetty-12-plus-core/pom.xml new file mode 100644 index 0000000..70c70fd --- /dev/null +++ b/hbase-shaded-jetty-12-plus-core/pom.xml @@ -0,0 +1,223 @@ + + + + 4.0.0 + + org.apache.hbase.thirdparty + hbase-thirdparty + ${revision} + .. + + hbase-shaded-jetty-12-plus-core + Apache HBase Relocated (Shaded) Jetty 12+ Libs: Core + Pulls down Jetty core, relocates it and makes a far jar. + + + org.eclipse.jetty + jetty-server + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-http + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-util + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-io + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-jmx + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-util-ajax + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-security + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-session + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-ee + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-xml + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + + + + maven-clean-plugin + + + + ${basedir} + + dependency-reduced-pom.xml + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + true + true + + + org.eclipse.jetty + ${rename.offset}.org.eclipse.jetty + + + + -org.eclipse.jetty + -${rename.offset}.org.eclipse.jetty + + + + + + org.slf4j:slf4j-api + + org.apache.hbase.thirdparty:hbase-shaded-jetty-12-plus-core + + + + + + false + + + + + + + + + + diff --git a/hbase-shaded-jetty-12-plus-ee8/pom.xml b/hbase-shaded-jetty-12-plus-ee8/pom.xml new file mode 100644 index 0000000..e6cf5e6 --- /dev/null +++ b/hbase-shaded-jetty-12-plus-ee8/pom.xml @@ -0,0 +1,252 @@ + + + + 4.0.0 + + org.apache.hbase.thirdparty + hbase-thirdparty + ${revision} + .. + + hbase-shaded-jetty-12-plus-ee8 + Apache HBase Relocated (Shaded) Jetty 12+ Libs: EE8 + Pulls down Jetty EE8, relocates it and makes a far jar. + + + org.eclipse.jetty.ee8 + jetty-ee8-servlet + ${jetty-12-plus.version} + + + org.eclipse.jetty + servlet-api + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty.ee8 + jetty-ee8-security + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty.ee8 + jetty-ee8-nested + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty.ee8 + jetty-ee8-webapp + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + + org.eclipse.jetty + jetty-server + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-http + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-util + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-io + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-jmx + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-util-ajax + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-security + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-session + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-ee + ${jetty-12-plus.version} + provided + + + org.eclipse.jetty + jetty-xml + ${jetty-12-plus.version} + provided + + + + + + maven-clean-plugin + + + + ${basedir} + + dependency-reduced-pom.xml + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + true + true + + + org.eclipse.jetty + ${rename.offset}.org.eclipse.jetty + + + + -org.eclipse.jetty + -${rename.offset}.org.eclipse.jetty + + + + + + org.slf4j:slf4j-api + + org.apache.hbase.thirdparty:hbase-shaded-jetty-12-plus-ee8 + + + + + + false + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + relocate-web-xml + + run + + package + + + + + + + org.eclipse.jetty + ${rename.offset}.org.eclipse.jetty + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 00962ab..569fbc5 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,8 @@ hbase-shaded-gson hbase-shaded-miscellaneous hbase-shaded-jetty + hbase-shaded-jetty-12-plus-core + hbase-shaded-jetty-12-plus-ee8 hbase-shaded-jersey hbase-shaded-jackson-jaxrs-json-provider hbase-noop-htrace @@ -139,6 +141,7 @@ 2.38.0 2.13.1 9.4.57.v20241219 + 12.0.22 3.1.0 2.46 2.6.1 From cbcf5236cc465269efe19ac53ce55a43d1198717 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 10 Jul 2025 20:26:33 +0530 Subject: [PATCH 02/13] Add dual JDK build support with configurable toolchains for Jetty 12 modules Implement Maven toolchains-based build system to support both JDK 8 and JDK 17 compilation in a single release with configurable setup: - Add toolchains plugin configuration for automatic JDK selection - Configure Jetty 12 modules (hbase-shaded-jetty-12-plus-*) to use JDK 17 - Maintain JDK 8 compatibility for existing modules - Split maven-enforcer-plugin into separate JDK 8/17 bytecode checks - Add generate-toolchains.sh script for dynamic toolchains.xml generation - Support JAVA8_HOME/JAVA17_HOME environment variable configuration - Auto-detect Java installations at /usr/lib/jvm/java-8 and /usr/lib/jvm/java-17 for CI - Force explicit environment variable setup for local development - Remove dependency on ~/.m2/toolchains.xml for project-specific configuration - Add documentation with setup commands and CI/local instructions This allows HBase 2.x to continue using JDK 8 compatible Jetty 9 modules while HBase 3.x can leverage JDK 17 Jetty 12 modules, with seamless Jenkins/CI builds and clear local development setup, eliminating the need for separate branches or releases. --- .gitignore | 1 + README.md | 62 ++++++++++++++ dev-support/generate-toolchains.sh | 94 +++++++++++++++++++++ hbase-shaded-jetty-12-plus-core/pom.xml | 39 +++++---- hbase-shaded-jetty-12-plus-ee8/pom.xml | 9 ++ pom.xml | 105 +++++++++++++++++++----- 6 files changed, 276 insertions(+), 34 deletions(-) create mode 100755 dev-support/generate-toolchains.sh diff --git a/.gitignore b/.gitignore index 8f1e942..f4b640b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ hbase-shaded-protobuf/src/main/resources .idea *.iml .flattened-pom.xml +toolchains.xml diff --git a/README.md b/README.md index 1fa036d..0768c54 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,66 @@ modern JDKs. Due to a bug in JDK, we cannot generate this code using a more modern version of the JDK. See [HBASE-26773](https://issues.apache.org/jira/browse/HBASE-26773) for details. +## Jetty 12 and Dual JDK Requirement + +Starting with version 4.1.12, this project requires JDK 8 and JDK 17 to accommodate different HBase versions: + +- **HBase 2.x**: Uses JDK 8 compatible modules (including `hbase-shaded-jetty` with Jetty 9) +- **HBase 3.x**: Uses JDK 17 modules (including `hbase-shaded-jetty-12-plus-*` with Jetty 12) + +### Why Dual JDK Support? + +Jetty 12 requires JDK 17 for compilation, but HBase 2.x deployments cannot move to Jetty 12 for JDK 8 compatibility. Our solution provides a single release containing modules for both JDK versions, eliminating the need for separate branches or releases. + +### Local Development Setup + +1. **Install both JDK versions**: JDK 8 and JDK 17 +2. **Set environment variables**: + ```sh + export JAVA8_HOME=/path/to/your/jdk8 + export JAVA17_HOME=/path/to/your/jdk17 + ``` +3. **Choose your toolchain setup approach** (see options below) + +#### Toolchain Setup Options + +**Option 1: Project-local toolchains.xml (Recommended)** +```sh +# Generate and use project-specific toolchains +export JAVA8_HOME=/path/to/your/jdk8 +export JAVA17_HOME=/path/to/your/jdk17 + ./dev-support/generate-toolchains.sh +mvn clean install -t toolchains.xml +``` + +**Option 2: Global Maven toolchains setup** +```sh +# Setup toolchains in ~/.m2/ directory +export JAVA8_HOME=/path/to/your/jdk8 +export JAVA17_HOME=/path/to/your/jdk17 + ./dev-support/generate-toolchains.sh +cp toolchains.xml ~/.m2/toolchains.xml +mvn clean install +``` + +**Why toolchains are required:** Maven needs explicit toolchain configuration to automatically select JDK 8 for legacy modules and JDK 17 for Jetty 12 modules. Environment variables alone are insufficient. + +### CI/Jenkins Setup + +For Jenkins/CI environments, the project auto-detects Java installations at standard paths: +- Java 8: `/usr/lib/jvm/java-8` +- Java 17: `/usr/lib/jvm/java-17` + +Simply run the toolchain generation script and build: +```sh +./generate-toolchains.sh +mvn clean install -t ./toolchains.xml +``` + +### Files +- `generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths +- `toolchains.xml` - Generated Maven toolchains configuration file (not checked in) + ## Build/Deploy To build, make sure that your environment uses JDK8, then just run: @@ -100,6 +160,8 @@ To build, make sure that your environment uses JDK8, then just run: mvn clean package ``` +**Note**: If you have Maven toolchains configured (recommended), the build will automatically use the appropriate JDK version for each module. + ## Release To cut a release candidate, update JIRA. The hbase-thirdparty currently uses diff --git a/dev-support/generate-toolchains.sh b/dev-support/generate-toolchains.sh new file mode 100755 index 0000000..761bea1 --- /dev/null +++ b/dev-support/generate-toolchains.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Script to generate toolchains.xml with configurable Java paths + +# Set default paths if environment variables are not set +if [ -z "$JAVA8_HOME" ]; then + echo "Trying to detect Java 8 installation (Jenkins/CI standard path)" + if [ -d "/usr/lib/jvm/java-8" ]; then + JAVA8_HOME="/usr/lib/jvm/java-8" + else + echo "Warning: JAVA8_HOME not set and Java 8 not found at /usr/lib/jvm/java-8" + echo "For local development, please set JAVA8_HOME environment variable" + JAVA8_HOME="/usr/lib/jvm/java-8" + fi +fi + +if [ -z "$JAVA17_HOME" ]; then + echo "Trying to detect Java 17 installation (Jenkins/CI standard path)" + if [ -d "/usr/lib/jvm/java-17" ]; then + JAVA17_HOME="/usr/lib/jvm/java-17" + else + echo "Warning: JAVA17_HOME not set and Java 17 not found at /usr/lib/jvm/java-17" + echo "For local development, please set JAVA17_HOME environment variable" + JAVA17_HOME="/usr/lib/jvm/java-17" + fi +fi + +echo "Generating toolchains.xml with:" +echo " JAVA8_HOME: $JAVA8_HOME" +echo " JAVA17_HOME: $JAVA17_HOME" + +# Generate toolchains.xml +cat > toolchains.xml << EOF + + + + + jdk + + 1.8 + + + $JAVA8_HOME + + + + jdk + + 17 + + + $JAVA17_HOME + + + +EOF + +echo "toolchains.xml generated successfully!" diff --git a/hbase-shaded-jetty-12-plus-core/pom.xml b/hbase-shaded-jetty-12-plus-core/pom.xml index 70c70fd..c4a5fe3 100644 --- a/hbase-shaded-jetty-12-plus-core/pom.xml +++ b/hbase-shaded-jetty-12-plus-core/pom.xml @@ -38,6 +38,15 @@ hbase-shaded-jetty-12-plus-core Apache HBase Relocated (Shaded) Jetty 12+ Libs: Core Pulls down Jetty core, relocates it and makes a far jar. + + + 17 + + ${compileSource} + + true + false + org.eclipse.jetty @@ -127,21 +136,21 @@ - - org.eclipse.jetty - jetty-ee - ${jetty-12-plus.version} - - - org.slf4j - slf4j-api - - - - org.eclipse.jetty - jetty-xml - ${jetty-12-plus.version} + org.eclipse.jetty + jetty-ee + ${jetty-12-plus.version} + + + org.slf4j + slf4j-api + + + + + org.eclipse.jetty + jetty-xml + ${jetty-12-plus.version} org.slf4j @@ -200,7 +209,7 @@ produce. See below for how to exclusion of transitive dependencies. --> org.slf4j:slf4j-api - diff --git a/hbase-shaded-jetty-12-plus-ee8/pom.xml b/hbase-shaded-jetty-12-plus-ee8/pom.xml index e6cf5e6..e58b723 100644 --- a/hbase-shaded-jetty-12-plus-ee8/pom.xml +++ b/hbase-shaded-jetty-12-plus-ee8/pom.xml @@ -38,6 +38,15 @@ hbase-shaded-jetty-12-plus-ee8 Apache HBase Relocated (Shaded) Jetty 12+ Libs: EE8 Pulls down Jetty EE8, relocates it and makes a far jar. + + + 17 + + ${compileSource} + + true + false + org.eclipse.jetty.ee8 diff --git a/pom.xml b/pom.xml index 569fbc5..3fe6cfb 100644 --- a/pom.xml +++ b/pom.xml @@ -127,8 +127,15 @@ true yyyy-MM-dd'T'HH:mm ${maven.build.timestamp} + 1.8 ${compileSource} + + ${compileSource} + + false + true + 3.3.3 1.7.1 org.apache.hbase.thirdparty @@ -250,8 +257,8 @@ maven-compiler-plugin 3.11.0 - 1.8 - 1.8 + ${compileSource} + ${compileSource} @@ -279,6 +286,11 @@ maven-enforcer-plugin 3.4.1 + + org.apache.maven.plugins + maven-toolchains-plugin + 3.2.0 + org.codehaus.mojo flatten-maven-plugin @@ -300,6 +312,24 @@ + + org.apache.maven.plugins + maven-toolchains-plugin + + + + ${toolchain.jdk.version} + + + + + + + toolchain + + + + org.codehaus.mojo build-helper-maven-plugin @@ -522,28 +552,65 @@ org.apache.maven.plugins maven-enforcer-plugin - - - - ${compileSource} - - module-info - **.glassfish.jersey.internal.jsr166.SubmissionPublisher* - **.glassfish.jersey.internal.jsr166.JerseyFlowSubscriber* - - true - - test - - HBase has unsupported dependencies. + + + + enforce-jdk8-bytecode + + enforce + + verify + + + + ${compileSource} + + module-info + **.glassfish.jersey.internal.jsr166.SubmissionPublisher* + **.glassfish.jersey.internal.jsr166.JerseyFlowSubscriber* + + true + + test + + HBase has unsupported dependencies. HBase requires that all dependencies be compiled with version ${compileSource} or earlier of the JDK to properly build from source. You appear to be using a newer dependency. You can use either "mvn -version" or "mvn enforcer:display-info" to verify what version is active. Non-release builds can temporarily build with a newer JDK version by setting the 'compileSource' property (eg. mvn -DcompileSource=1.8 clean package). - - - + + + ${skipJdk8BytecodeCheck} + + + + + enforce-jdk17-bytecode + + enforce + + verify + + + + 17 + + module-info + + true + + test + + Some modules require JDK 17 bytecode compatibility. + For example, Jetty 12 modules (hbase-shaded-jetty-12-plus-*) require dependencies compiled with JDK 17. + These modules are intended for HBase 3.x which supports JDK 17+. + + + ${skipJdk17BytecodeCheck} + + + From f1b5d2f98a721e82bb14390eb6851b091fe79f70 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 10 Jul 2025 20:27:24 +0530 Subject: [PATCH 03/13] WIP: Jenkins change --- dev-support/jenkins/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile index 00649d0..9fdeb40 100644 --- a/dev-support/jenkins/Jenkinsfile +++ b/dev-support/jenkins/Jenkinsfile @@ -53,7 +53,9 @@ pipeline { DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile" YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh" ARCHIVE_PATTERN_LIST = '*.dump' - SET_JAVA_HOME = '/opt/java/openjdk' + SET_JAVA_HOME = "/usr/lib/jvm/java-8" + JAVA8_HOME = "/usr/lib/jvm/java-8" + JAVA17_HOME = "/usr/lib/jvm/java-17 PLUGINS = 'all' } From 7d7b2ef860143d1c0fcc5af2352d0eb2992d6c45 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 10 Jul 2025 20:37:30 +0530 Subject: [PATCH 04/13] Update README.md --- README.md | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0768c54..85e753f 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,15 @@ Starting with version 4.1.12, this project requires JDK 8 and JDK 17 to accommod Jetty 12 requires JDK 17 for compilation, but HBase 2.x deployments cannot move to Jetty 12 for JDK 8 compatibility. Our solution provides a single release containing modules for both JDK versions, eliminating the need for separate branches or releases. +### Why toolchains are required? +Maven needs explicit toolchain configuration to automatically select JDK 8 for existing modules and JDK 17 for Jetty 12 modules. Environment variables alone are insufficient. + +### Files +- `generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths +- `toolchains.xml` - Generated Maven toolchains configuration file (not checked in) + +## Build/Deploy + ### Local Development Setup 1. **Install both JDK versions**: JDK 8 and JDK 17 @@ -113,14 +122,15 @@ Jetty 12 requires JDK 17 for compilation, but HBase 2.x deployments cannot move ``` 3. **Choose your toolchain setup approach** (see options below) -#### Toolchain Setup Options +### Toolchain Setup Options -**Option 1: Project-local toolchains.xml (Recommended)** +**Option 1: Project-local toolchains.xml** ```sh # Generate and use project-specific toolchains export JAVA8_HOME=/path/to/your/jdk8 export JAVA17_HOME=/path/to/your/jdk17 - ./dev-support/generate-toolchains.sh +./dev-support/generate-toolchains.sh + mvn clean install -t toolchains.xml ``` @@ -129,13 +139,12 @@ mvn clean install -t toolchains.xml # Setup toolchains in ~/.m2/ directory export JAVA8_HOME=/path/to/your/jdk8 export JAVA17_HOME=/path/to/your/jdk17 - ./dev-support/generate-toolchains.sh +./dev-support/generate-toolchains.sh cp toolchains.xml ~/.m2/toolchains.xml + mvn clean install ``` -**Why toolchains are required:** Maven needs explicit toolchain configuration to automatically select JDK 8 for legacy modules and JDK 17 for Jetty 12 modules. Environment variables alone are insufficient. - ### CI/Jenkins Setup For Jenkins/CI environments, the project auto-detects Java installations at standard paths: @@ -148,19 +157,6 @@ Simply run the toolchain generation script and build: mvn clean install -t ./toolchains.xml ``` -### Files -- `generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths -- `toolchains.xml` - Generated Maven toolchains configuration file (not checked in) - -## Build/Deploy - -To build, make sure that your environment uses JDK8, then just run: - -```sh -mvn clean package -``` - -**Note**: If you have Maven toolchains configured (recommended), the build will automatically use the appropriate JDK version for each module. ## Release From 86ed68c745ebfebf6f1400f72b6e19638c3b42d4 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Thu, 10 Jul 2025 21:27:06 +0530 Subject: [PATCH 05/13] Fix double quotes in Jenkinfile --- dev-support/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile index 9fdeb40..03adf5b 100644 --- a/dev-support/jenkins/Jenkinsfile +++ b/dev-support/jenkins/Jenkinsfile @@ -55,7 +55,7 @@ pipeline { ARCHIVE_PATTERN_LIST = '*.dump' SET_JAVA_HOME = "/usr/lib/jvm/java-8" JAVA8_HOME = "/usr/lib/jvm/java-8" - JAVA17_HOME = "/usr/lib/jvm/java-17 + JAVA17_HOME = "/usr/lib/jvm/java-17" PLUGINS = 'all' } From f729cc467f8fa72470a2b90c15c278d9a5827007 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 15 Jul 2025 11:41:35 +0530 Subject: [PATCH 06/13] Use JDK17 as default, restrict JDK8 to hbase-unsafe module --- hbase-shaded-jetty-12-plus-core/pom.xml | 9 ------ hbase-shaded-jetty-12-plus-ee8/pom.xml | 9 ------ hbase-unsafe/pom.xml | 8 +++++ pom.xml | 39 +++++++++++++++++++------ 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/hbase-shaded-jetty-12-plus-core/pom.xml b/hbase-shaded-jetty-12-plus-core/pom.xml index c4a5fe3..fe2de51 100644 --- a/hbase-shaded-jetty-12-plus-core/pom.xml +++ b/hbase-shaded-jetty-12-plus-core/pom.xml @@ -38,15 +38,6 @@ hbase-shaded-jetty-12-plus-core Apache HBase Relocated (Shaded) Jetty 12+ Libs: Core Pulls down Jetty core, relocates it and makes a far jar. - - - 17 - - ${compileSource} - - true - false - org.eclipse.jetty diff --git a/hbase-shaded-jetty-12-plus-ee8/pom.xml b/hbase-shaded-jetty-12-plus-ee8/pom.xml index e58b723..e6cf5e6 100644 --- a/hbase-shaded-jetty-12-plus-ee8/pom.xml +++ b/hbase-shaded-jetty-12-plus-ee8/pom.xml @@ -38,15 +38,6 @@ hbase-shaded-jetty-12-plus-ee8 Apache HBase Relocated (Shaded) Jetty 12+ Libs: EE8 Pulls down Jetty EE8, relocates it and makes a far jar. - - - 17 - - ${compileSource} - - true - false - org.eclipse.jetty.ee8 diff --git a/hbase-unsafe/pom.xml b/hbase-unsafe/pom.xml index 4781911..f67e712 100644 --- a/hbase-unsafe/pom.xml +++ b/hbase-unsafe/pom.xml @@ -41,6 +41,14 @@ 3.1.0 8.29 + + ${java.min.version} + ${compileSource} + + false + true + + diff --git a/pom.xml b/pom.xml index 3fe6cfb..dab6a47 100644 --- a/pom.xml +++ b/pom.xml @@ -128,13 +128,13 @@ yyyy-MM-dd'T'HH:mm ${maven.build.timestamp} - 1.8 - ${compileSource} + 17 + 1.8 ${compileSource} - false - true + true + false 3.3.3 1.7.1 @@ -158,6 +158,27 @@ 3.30.2-GA 2.19.0 2.30.0 + + + -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true + --add-modules jdk.unsupported + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED + --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-exports java.base/sun.net.dns=ALL-UNNAMED + --add-exports java.base/sun.net.util=ALL-UNNAMED + --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED + --add-opens java.base/sun.security.x509=ALL-UNNAMED + --add-opens java.base/sun.security.util=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + ${hbase-surefire.jdk17.flags} @@ -553,7 +574,7 @@ org.apache.maven.plugins maven-enforcer-plugin - + enforce-jdk8-bytecode @@ -563,7 +584,7 @@ - ${compileSource} + ${java.min.version} module-info **.glassfish.jersey.internal.jsr166.SubmissionPublisher* @@ -574,7 +595,7 @@ test HBase has unsupported dependencies. - HBase requires that all dependencies be compiled with version ${compileSource} or earlier + hbase-unsafe requires to be compiled with version 1.8 of the JDK to properly build from source. You appear to be using a newer dependency. You can use either "mvn -version" or "mvn enforcer:display-info" to verify what version is active. Non-release builds can temporarily build with a newer JDK version by setting the @@ -584,7 +605,7 @@ ${skipJdk8BytecodeCheck} - + enforce-jdk17-bytecode @@ -594,7 +615,7 @@ - 17 + ${compileSource} module-info From 53eb62d1ae96aee9aea3fce3555ad463bfafabde Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 15 Jul 2025 11:55:47 +0530 Subject: [PATCH 07/13] Bump yetus to be same as in hbase, and try jenkins change to generate toolchains.xml and copy to ~/.m2 --- dev-support/jenkins/Dockerfile | 2 +- dev-support/jenkins/Jenkinsfile | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 9119a2b..3d28df8 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -17,7 +17,7 @@ # Dockerfile for hbase-operator-tools pre-commit build. # https://builds.apache.org/job/PreCommit-HBASE-OPERATOR-TOOLS-Build -FROM maven:3.9-eclipse-temurin-8 +FROM maven:3.9-eclipse-temurin-17 # hadolint ignore=DL3008 RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile index 03adf5b..9bc1d3f 100644 --- a/dev-support/jenkins/Jenkinsfile +++ b/dev-support/jenkins/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile" YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh" ARCHIVE_PATTERN_LIST = '*.dump' - SET_JAVA_HOME = "/usr/lib/jvm/java-8" + SET_JAVA_HOME = "/usr/lib/jvm/java-17" JAVA8_HOME = "/usr/lib/jvm/java-8" JAVA17_HOME = "/usr/lib/jvm/java-17" PLUGINS = 'all' @@ -149,7 +149,7 @@ pipeline { } tools { // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in. - jdk "jdk_1.8_latest" + jdk "jdk_17_latest" } stages { stage ('setup') { @@ -165,6 +165,17 @@ pipeline { dir ("component") { echo 'Build the htrace replacement artifact.' checkout scm + sh label: 'Generate and setup toolchains', script: '''#!/bin/bash -e + echo "Generating toolchains.xml" + ./dev-support/generate-toolchains.sh + + echo "Setting up Maven toolchains" + mkdir -p ~/.m2 + cp toolchains.xml ~/.m2/toolchains.xml + + echo "Toolchains setup complete" + cat ~/.m2/toolchains.xml + ''' sh 'mvn -DskipTests -pl hbase-noop-htrace clean package' } dir ("tools") { From f566a926da860458032ec6e18e636a34f63c0fe8 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 15 Jul 2025 16:06:32 +0530 Subject: [PATCH 08/13] Update Dockerfile to support JDK 8 and HDK 17 --- dev-support/jenkins/Dockerfile | 104 ++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 3d28df8..09999a9 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -14,21 +14,97 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Dockerfile for hbase-operator-tools pre-commit build. -# https://builds.apache.org/job/PreCommit-HBASE-OPERATOR-TOOLS-Build - -FROM maven:3.9-eclipse-temurin-17 - -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - binutils \ - git \ - rsync \ - shellcheck \ - patch \ - wget && \ +# Dockerfile for hbase-thirdparty pre-commit build. +# +# +# Built in multiple stages so as to avoid re-downloading large binaries when +# tweaking unrelated aspects of the image. +FROM ubuntu:22.04 AS base_image +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ + DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + ca-certificates=20211016 \ + curl='7.81.0-*' \ + locales='2.35-*' \ + bash='5.1-*' \ + binutils='2.38-*' \ + build-essential=12.9ubuntu3 \ + git='1:2.34.1-*' \ + rsync='3.2.3-*' \ + tar='1.34+dfsg-*' \ + wget='1.21.2-*' \ + shellcheck='0.8.0-*' \ + patch='2.7.6-*' \ + && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* \ + && \ + locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 + +## +# download sundry dependencies +# + +FROM base_image AS maven_download_image +ENV MAVEN_VERSION='3.9.8' +ENV MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" +ENV MAVEN_SHA512='7d171def9b85846bf757a2cec94b7529371068a0670df14682447224e57983528e97a6d1b850327e4ca02b139abaab7fcb93c4315119e6f0ffb3f0cbc0d0b9a2' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/maven.tar.gz "${MAVEN_URL}" && \ + echo "${MAVEN_SHA512} */tmp/maven.tar.gz" | sha512sum -c - + +FROM base_image AS openjdk8_download_image +ENV OPENJDK8_URL='https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u412-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u412b08.tar.gz' +ENV OPENJDK8_SHA256='b9884a96f78543276a6399c3eb8c2fd8a80e6b432ea50e87d3d12d495d1d2808' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk8.tar.gz "${OPENJDK8_URL}" && \ + echo "${OPENJDK8_SHA256} */tmp/adoptopenjdk8.tar.gz" | sha256sum -c - + +FROM base_image AS openjdk17_download_image +ENV OPENJDK17_URL='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz' +ENV OPENJDK17_SHA256='aa7fb6bb342319d227a838af5c363bfa1b4a670c209372f9e6585bd79da6220c' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \ + echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c - + +## +# build the final image +# + +FROM base_image +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3010 +COPY --from=maven_download_image /tmp/maven.tar.gz /tmp/maven.tar.gz +RUN tar xzf /tmp/maven.tar.gz -C /opt && \ + ln -s "/opt/$(dirname "$(tar -tf /tmp/maven.tar.gz | head -n1)")" /opt/maven && \ + rm /tmp/maven.tar.gz + +## +# ensure JVMs are available under `/usr/lib/jvm` and prefix each installation +# as `java-` so as to conform with Yetus's assumptions. +# + +# hadolint ignore=DL3010 +COPY --from=openjdk8_download_image /tmp/adoptopenjdk8.tar.gz /tmp/adoptopenjdk8.tar.gz +RUN mkdir -p /usr/lib/jvm && \ + tar xzf /tmp/adoptopenjdk8.tar.gz -C /usr/lib/jvm && \ + ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk8.tar.gz | head -n1)")" /usr/lib/jvm/java-8-adoptopenjdk && \ + ln -s /usr/lib/jvm/java-8-adoptopenjdk /usr/lib/jvm/java-8 && \ + rm /tmp/adoptopenjdk8.tar.gz + +# hadolint ignore=DL3010 +COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz /tmp/adoptopenjdk17.tar.gz +RUN mkdir -p /usr/lib/jvm && \ + tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \ + ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head -n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \ + ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \ + rm /tmp/adoptopenjdk17.tar.gz + +# configure default environment for Yetus +ENV MAVEN_HOME='/opt/maven' CMD ["/bin/bash"] From a684c3779de27c5db3b8da7909795d8c0e45096a Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Sat, 19 Jul 2025 02:35:17 +0530 Subject: [PATCH 09/13] Try Dockerfile change and make release target releated changes --- dev-support/generate-toolchains.sh | 58 ++++++------------------- dev-support/jenkins/Dockerfile | 15 +++++++ dev-support/jenkins/Jenkinsfile | 11 ----- dev-support/toolchains-jenkins.xml | 41 +++++++++++++++++ hbase-shaded-jetty-12-plus-core/pom.xml | 5 +++ hbase-shaded-jetty-12-plus-ee8/pom.xml | 5 +++ hbase-shaded-protobuf/pom.xml | 7 +++ hbase-unsafe/pom.xml | 6 +-- pom.xml | 20 ++++----- 9 files changed, 97 insertions(+), 71 deletions(-) create mode 100644 dev-support/toolchains-jenkins.xml diff --git a/dev-support/generate-toolchains.sh b/dev-support/generate-toolchains.sh index 761bea1..be764c1 100755 --- a/dev-support/generate-toolchains.sh +++ b/dev-support/generate-toolchains.sh @@ -41,54 +41,22 @@ if [ -z "$JAVA17_HOME" ]; then fi fi +TEMPLATE_FILE="./dev-support/toolchains-jenkins.xml" +OUTPUT_FILE="toolchains.xml" + +if [ ! -f "$TEMPLATE_FILE" ]; then + echo "Template file $TEMPLATE_FILE not found!" + exit 1 +fi + echo "Generating toolchains.xml with:" echo " JAVA8_HOME: $JAVA8_HOME" echo " JAVA17_HOME: $JAVA17_HOME" -# Generate toolchains.xml -cat > toolchains.xml << EOF - - - - - jdk - - 1.8 - - - $JAVA8_HOME - - - - jdk - - 17 - - - $JAVA17_HOME - - - -EOF +# Substitute placeholders and write output +sed \ + -e "s|/usr/lib/jvm/java-8|$JAVA8_HOME|g" \ + -e "s|/usr/lib/jvm/java-17|$JAVA17_HOME|g" \ + "$TEMPLATE_FILE" > "$OUTPUT_FILE" echo "toolchains.xml generated successfully!" diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 09999a9..98b69e7 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -105,6 +105,21 @@ RUN mkdir -p /usr/lib/jvm && \ # configure default environment for Yetus ENV MAVEN_HOME='/opt/maven' + +# create a wrapper for mvn to always pass the toolchains.xml file +RUN mv /opt/maven/bin/mvn /opt/maven/bin/mvn-original && \ + cat > /opt/maven/bin/mvn <<'EOF' +#!/bin/bash +TOOLCHAIN="${BASEDIR}/dev-support/toolchains-jenkins.xml" +if [ -f "$TOOLCHAIN" ]; then + echo "Bypassed mvn and added: -t ${TOOLCHAIN} as OPTS!" + exec ${MAVEN_HOME}/bin/mvn-original "$@" -t "$TOOLCHAIN" +else + echo "Unexpected: ${TOOLCHAIN} file is missing!" + exec ${MAVEN_HOME}/bin/mvn-original "$@" +fi +EOF +RUN chmod +x /opt/maven/bin/mvn && cat /opt/maven/bin/mvn CMD ["/bin/bash"] diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile index 9bc1d3f..e6d9b97 100644 --- a/dev-support/jenkins/Jenkinsfile +++ b/dev-support/jenkins/Jenkinsfile @@ -165,17 +165,6 @@ pipeline { dir ("component") { echo 'Build the htrace replacement artifact.' checkout scm - sh label: 'Generate and setup toolchains', script: '''#!/bin/bash -e - echo "Generating toolchains.xml" - ./dev-support/generate-toolchains.sh - - echo "Setting up Maven toolchains" - mkdir -p ~/.m2 - cp toolchains.xml ~/.m2/toolchains.xml - - echo "Toolchains setup complete" - cat ~/.m2/toolchains.xml - ''' sh 'mvn -DskipTests -pl hbase-noop-htrace clean package' } dir ("tools") { diff --git a/dev-support/toolchains-jenkins.xml b/dev-support/toolchains-jenkins.xml new file mode 100644 index 0000000..88bb0fe --- /dev/null +++ b/dev-support/toolchains-jenkins.xml @@ -0,0 +1,41 @@ + + + + + jdk + + 1.8 + + + /usr/lib/jvm/java-8 + + + + jdk + + 17 + + + /usr/lib/jvm/java-17 + + + diff --git a/hbase-shaded-jetty-12-plus-core/pom.xml b/hbase-shaded-jetty-12-plus-core/pom.xml index fe2de51..5756391 100644 --- a/hbase-shaded-jetty-12-plus-core/pom.xml +++ b/hbase-shaded-jetty-12-plus-core/pom.xml @@ -38,6 +38,11 @@ hbase-shaded-jetty-12-plus-core Apache HBase Relocated (Shaded) Jetty 12+ Libs: Core Pulls down Jetty core, relocates it and makes a far jar. + + + true + false + org.eclipse.jetty diff --git a/hbase-shaded-jetty-12-plus-ee8/pom.xml b/hbase-shaded-jetty-12-plus-ee8/pom.xml index e6cf5e6..750ed72 100644 --- a/hbase-shaded-jetty-12-plus-ee8/pom.xml +++ b/hbase-shaded-jetty-12-plus-ee8/pom.xml @@ -38,6 +38,11 @@ hbase-shaded-jetty-12-plus-ee8 Apache HBase Relocated (Shaded) Jetty 12+ Libs: EE8 Pulls down Jetty EE8, relocates it and makes a far jar. + + + true + false + org.eclipse.jetty.ee8 diff --git a/hbase-shaded-protobuf/pom.xml b/hbase-shaded-protobuf/pom.xml index 6ecbc35..8b3ff5d 100644 --- a/hbase-shaded-protobuf/pom.xml +++ b/hbase-shaded-protobuf/pom.xml @@ -29,6 +29,13 @@ hbase-shaded-protobuf Apache HBase Patched and Relocated (Shaded) Protobuf Pulls down protobuf, patches it, compiles, and then relocates/shades. + + + ${java.min.version} + diff --git a/hbase-unsafe/pom.xml b/hbase-unsafe/pom.xml index f67e712..f2fd567 100644 --- a/hbase-unsafe/pom.xml +++ b/hbase-unsafe/pom.xml @@ -42,11 +42,7 @@ 3.1.0 8.29 - ${java.min.version} - ${compileSource} - - false - true + ${java.min.version} diff --git a/pom.xml b/pom.xml index dab6a47..b474385 100644 --- a/pom.xml +++ b/pom.xml @@ -128,13 +128,17 @@ yyyy-MM-dd'T'HH:mm ${maven.build.timestamp} - 17 1.8 + 17 + 8 + ${releaseTarget} + ${releaseTarget} + ${releaseTarget} - ${compileSource} + ${java.secondary.version} - true - false + false + true 3.3.3 1.7.1 @@ -276,11 +280,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 - - ${compileSource} - ${compileSource} - + 3.13.0 org.apache.maven.plugins @@ -615,7 +615,7 @@ - ${compileSource} + ${java.secondary.version} module-info From 681e54d9c07291f599a314c26b8b111b01a4b713 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 22 Jul 2025 12:25:35 +0530 Subject: [PATCH 10/13] Address review coments --- README.md | 79 +++++++++++++++++++++++++++-------- hbase-shaded-protobuf/pom.xml | 2 +- hbase-unsafe/pom.xml | 2 +- pom.xml | 20 ++++----- 4 files changed, 73 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 85e753f..cf4c0c7 100644 --- a/README.md +++ b/README.md @@ -86,25 +86,35 @@ In case build fails due to protobuf-java version change, we can follow below ste --- -Note that this project requires JDK8. This is because a bunch of the code we -have in hbase-unsafe is using old APIs that have been removed from more -modern JDKs. Due to a bug in JDK, we cannot generate this code using a more -modern version of the JDK. See -[HBASE-26773](https://issues.apache.org/jira/browse/HBASE-26773) for details. +## Dual JDK Requirements -## Jetty 12 and Dual JDK Requirement - -Starting with version 4.1.12, this project requires JDK 8 and JDK 17 to accommodate different HBase versions: +Starting with version 4.1.12, this project requires both JDK 8 and JDK 17 to accommodate different HBase versions: - **HBase 2.x**: Uses JDK 8 compatible modules (including `hbase-shaded-jetty` with Jetty 9) - **HBase 3.x**: Uses JDK 17 modules (including `hbase-shaded-jetty-12-plus-*` with Jetty 12) -### Why Dual JDK Support? - Jetty 12 requires JDK 17 for compilation, but HBase 2.x deployments cannot move to Jetty 12 for JDK 8 compatibility. Our solution provides a single release containing modules for both JDK versions, eliminating the need for separate branches or releases. +### JDK 8 Required Modules + +This project has specific JDK requirements for different modules: + +- **hbase-unsafe**: Must be built with JDK 8 because it uses old APIs that have been removed from more modern JDKs. Due to a bug in JDK, we cannot generate this code using a more modern version of the JDK. See [HBASE-26773](https://issues.apache.org/jira/browse/HBASE-26773) for details. +- **hbase-shaded-protobuf**: Must be built with JDK 8 because it depends on internal Java APIs such as `sun.misc.Unsafe`. These internal APIs are inaccessible when compiling with newer JDKs but release target set to JDK 8. + +### JDK 17 Compilation with JDK 8 Target + +**All other modules** (including Jetty 12 modules) use JDK 17 for compilation but with release target set to JDK 8. + +### Maven Enforcer Plugin Profiles + +The project uses Maven enforcer plugin profiles to ensure these requirements are met: +- **enforce-jdk8-bytecode**: Ensures hbase-unsafe and hbase-shaded-protobuf are built with JDK 8. +- **enforce-jdk17-bytecode**: For **Jetty 12 modules** (hbase-shaded-jetty-12-plus-*), the enforcer plugin allows JDK 17 bytecode to be included from Jetty 12 dependencies, while still using JDK 8 as the release target. + ### Why toolchains are required? -Maven needs explicit toolchain configuration to automatically select JDK 8 for existing modules and JDK 17 for Jetty 12 modules. Environment variables alone are insufficient. + +Maven needs explicit toolchain configuration to automatically select JDK 8 for hbase-unsafe and hbase-shaded-protobuf modules, and JDK 17 for all other modules including Jetty 12 modules. Environment variables alone are insufficient. ### Files - `generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths @@ -147,16 +157,49 @@ mvn clean install ### CI/Jenkins Setup -For Jenkins/CI environments, the project auto-detects Java installations at standard paths: -- Java 8: `/usr/lib/jvm/java-8` -- Java 17: `/usr/lib/jvm/java-17` +The Jenkins CI environment uses a Docker-based build system that automatically handles the dual JDK requirements without any manual configuration. -Simply run the toolchain generation script and build: -```sh -./generate-toolchains.sh -mvn clean install -t ./toolchains.xml +#### Multi-JDK Docker Environment + +The Jenkins build uses a custom Dockerfile (`dev-support/jenkins/Dockerfile`) that: +- Downloads and installs both JDK 8 (Adoptium Temurin) and JDK 17 (Adoptium Temurin) +- Places them at standardized paths: + - Java 8: `/usr/lib/jvm/java-8` + - Java 17: `/usr/lib/jvm/java-17` +- Includes Maven 3.9.8 for the build process + +#### Maven Wrapper with Automatic Toolchains + +The Docker image creates a Maven wrapper that automatically handles toolchain configuration by replacing the original `mvn` command with a wrapper that always passes the `-t ${BASEDIR}/dev-support/toolchains-jenkins.xml` parameter to ensure the correct toolchains file is used for every Maven invocation. + +#### Automatic JDK Selection + +The system uses a pre-configured toolchains file (`dev-support/toolchains-jenkins.xml`) that: +- Defines JDK 1.8 toolchain pointing to `/usr/lib/jvm/java-8` +- Defines JDK 17 toolchain pointing to `/usr/lib/jvm/java-17` +- Allows Maven to automatically select the correct JDK for each module based on the toolchain requirements in their POMs + +#### Jenkins Build Process + +In Jenkins, the build process is completely automated: +1. Docker container starts with both JDKs pre-installed +2. Maven wrapper automatically passes the toolchains configuration +3. Each module uses the appropriate JDK version: + - `hbase-unsafe` and `hbase-shaded-protobuf`: Built with JDK 8 + - All other modules: Built with JDK 17 (with JDK 8 release target) +4. No manual toolchain setup or environment configuration required + +#### Environment Variables + +The Jenkinsfile sets the following environment variables: +```bash +SET_JAVA_HOME="/usr/lib/jvm/java-17" # Default JDK for the build +JAVA8_HOME="/usr/lib/jvm/java-8" # JDK 8 location +JAVA17_HOME="/usr/lib/jvm/java-17" # JDK 17 location ``` +This automated setup ensures consistent builds across all Jenkins jobs without requiring developers or maintainers to manually configure toolchains in the CI environment. + ## Release diff --git a/hbase-shaded-protobuf/pom.xml b/hbase-shaded-protobuf/pom.xml index 8b3ff5d..6443dae 100644 --- a/hbase-shaded-protobuf/pom.xml +++ b/hbase-shaded-protobuf/pom.xml @@ -34,7 +34,7 @@ hbase-shaded-protobuf must be built with JDK 8 because it depends on internal Java APIs such as sun.misc.Unsafe. These internal APIs are inaccessible when compiling with newer JDKs using the release setting in Maven. --> - ${java.min.version} + ${hbase.unsafe.and.protobuf.java.version} diff --git a/hbase-unsafe/pom.xml b/hbase-unsafe/pom.xml index f2fd567..98e864b 100644 --- a/hbase-unsafe/pom.xml +++ b/hbase-unsafe/pom.xml @@ -42,7 +42,7 @@ 3.1.0 8.29 - ${java.min.version} + ${hbase.unsafe.and.protobuf.java.version} diff --git a/pom.xml b/pom.xml index b474385..c33b66c 100644 --- a/pom.xml +++ b/pom.xml @@ -128,12 +128,12 @@ yyyy-MM-dd'T'HH:mm ${maven.build.timestamp} - 1.8 + 1.8 17 - 8 - ${releaseTarget} - ${releaseTarget} - ${releaseTarget} + 8 + ${java.release.version} + ${java.release.version} + ${java.release.version} ${java.secondary.version} @@ -400,9 +400,9 @@ - [${java.min.version},) + [${hbase.unsafe.and.protobuf.java.version},) Java is out of date. - HBase requires at least version ${java.min.version} of the JDK to properly build from source. + HBase requires at least version ${hbase.unsafe.and.protobuf.java.version} of the JDK to properly build from source. See the reference guide on building for more information: http://hbase.apache.org/book.html#build @@ -574,7 +574,7 @@ org.apache.maven.plugins maven-enforcer-plugin - + enforce-jdk8-bytecode @@ -584,7 +584,7 @@ - ${java.min.version} + ${hbase.unsafe.and.protobuf.java.version} module-info **.glassfish.jersey.internal.jsr166.SubmissionPublisher* @@ -605,7 +605,7 @@ ${skipJdk8BytecodeCheck} - + enforce-jdk17-bytecode From 39fac5c8cbd12235239071b31e8158c7b602bec6 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 22 Jul 2025 13:55:36 +0530 Subject: [PATCH 11/13] Fix spotless --- dev-support/jenkins/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 98b69e7..5ebc37d 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -105,14 +105,14 @@ RUN mkdir -p /usr/lib/jvm && \ # configure default environment for Yetus ENV MAVEN_HOME='/opt/maven' - + # create a wrapper for mvn to always pass the toolchains.xml file RUN mv /opt/maven/bin/mvn /opt/maven/bin/mvn-original && \ cat > /opt/maven/bin/mvn <<'EOF' #!/bin/bash TOOLCHAIN="${BASEDIR}/dev-support/toolchains-jenkins.xml" if [ -f "$TOOLCHAIN" ]; then - echo "Bypassed mvn and added: -t ${TOOLCHAIN} as OPTS!" + echo "Added: -t ${TOOLCHAIN} to mvn flags!" exec ${MAVEN_HOME}/bin/mvn-original "$@" -t "$TOOLCHAIN" else echo "Unexpected: ${TOOLCHAIN} file is missing!" From 1500c1d6d6b28f300e33e68e22abc5d9d8ab8639 Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 22 Jul 2025 14:08:13 +0530 Subject: [PATCH 12/13] Fix minor doc issues --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf4c0c7..ff026e5 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ The project uses Maven enforcer plugin profiles to ensure these requirements are Maven needs explicit toolchain configuration to automatically select JDK 8 for hbase-unsafe and hbase-shaded-protobuf modules, and JDK 17 for all other modules including Jetty 12 modules. Environment variables alone are insufficient. ### Files -- `generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths +- `dev-support/generate-toolchains.sh` - Script to generate toolchains.xml with configurable paths - `toolchains.xml` - Generated Maven toolchains configuration file (not checked in) ## Build/Deploy @@ -139,8 +139,11 @@ Maven needs explicit toolchain configuration to automatically select JDK 8 for h # Generate and use project-specific toolchains export JAVA8_HOME=/path/to/your/jdk8 export JAVA17_HOME=/path/to/your/jdk17 + +# Below command will generate toolchains.xml in project root ./dev-support/generate-toolchains.sh +# Run build by passing toolchains.xml file to maven mvn clean install -t toolchains.xml ``` @@ -149,9 +152,14 @@ mvn clean install -t toolchains.xml # Setup toolchains in ~/.m2/ directory export JAVA8_HOME=/path/to/your/jdk8 export JAVA17_HOME=/path/to/your/jdk17 + +# Below command will generate toolchains.xml in project root ./dev-support/generate-toolchains.sh + +# Copy the generated file to global .m2 directory cp toolchains.xml ~/.m2/toolchains.xml +# Run build as usual mvn clean install ``` From 0eb4072f31f359b5740d79861599335c7a4d1ddf Mon Sep 17 00:00:00 2001 From: Nihal Jain Date: Tue, 22 Jul 2025 17:39:31 +0530 Subject: [PATCH 13/13] Address review comments --- hbase-shaded-protobuf/pom.xml | 2 ++ hbase-unsafe/pom.xml | 4 +++- pom.xml | 8 +++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hbase-shaded-protobuf/pom.xml b/hbase-shaded-protobuf/pom.xml index 6443dae..89195df 100644 --- a/hbase-shaded-protobuf/pom.xml +++ b/hbase-shaded-protobuf/pom.xml @@ -35,6 +35,8 @@ sun.misc.Unsafe. These internal APIs are inaccessible when compiling with newer JDKs using the release setting in Maven. --> ${hbase.unsafe.and.protobuf.java.version} + ${java.release.version} + ${java.release.version} diff --git a/hbase-unsafe/pom.xml b/hbase-unsafe/pom.xml index 98e864b..64d32aa 100644 --- a/hbase-unsafe/pom.xml +++ b/hbase-unsafe/pom.xml @@ -43,7 +43,9 @@ 8.29 ${hbase.unsafe.and.protobuf.java.version} - + ${java.release.version} + ${java.release.version} + diff --git a/pom.xml b/pom.xml index c33b66c..d90b68a 100644 --- a/pom.xml +++ b/pom.xml @@ -129,13 +129,11 @@ ${maven.build.timestamp} 1.8 - 17 + 17 8 - ${java.release.version} - ${java.release.version} ${java.release.version} - ${java.secondary.version} + ${java.version} false true @@ -615,7 +613,7 @@ - ${java.secondary.version} + ${java.version} module-info