Skip to content

Commit ece0bd2

Browse files
Divneet18Divneet18
authored
Merge branch 'branch-2' into HBASE-28428-branch-2
2 parents 84a9729 + 1a99f65 commit ece0bd2

411 files changed

Lines changed: 9968 additions & 4280 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ The latest HBase can be downloaded from the [download page](https://hbase.apache
2828

2929
We use mailing lists to send notice and discuss. The mailing lists and archives are listed [here](http://hbase.apache.org/mail-lists.html)
3030

31+
We use the #hbase channel on the official [ASF Slack Workspace](https://the-asf.slack.com/) for real time questions and discussions. Please mail dev@hbase.apache.org to request an invite.
32+
3133
# How to Contribute
3234
The source code can be found at https://hbase.apache.org/source-repository.html
3335

‎dev-support/Jenkinsfile‎

Lines changed: 184 additions & 26 deletions
Large diffs are not rendered by default.

‎dev-support/Jenkinsfile_GitHub‎

Lines changed: 159 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,17 @@ pipeline {
3636
YETUS_REL = 'yetus'
3737
DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
3838
YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
39-
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
40-
YETUS_VERSION = 'rel/0.12.0'
39+
YETUS_VERSION = '0.15.0'
4140
GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
4241
JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
42+
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
43+
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
44+
BLANKS_EOL_IGNORE_FILE = 'dev-support/blanks-eol-ignore.txt'
45+
BLANKS_TABS_IGNORE_FILE = 'dev-support/blanks-tabs-ignore.txt'
4346
// output from surefire; sadly the archive function in yetus only works on file names.
4447
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
4548
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
46-
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
49+
TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
4750
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
4851
// set build parallel
4952
BUILD_THREAD = 4
@@ -55,6 +58,7 @@ pipeline {
5558
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
5659
WORKDIR_REL_JDK8_HADOOP2_CHECK = 'yetus-jdk8-hadoop2-check'
5760
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
61+
WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
5862
ASF_NIGHTLIES = 'https://nightlies.apache.org'
5963
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
6064
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
@@ -101,11 +105,11 @@ pipeline {
101105
checkout scm
102106
}
103107
dir("${YETUSDIR}") {
104-
checkout([
105-
$class : 'GitSCM',
106-
branches : [[name: "${YETUS_VERSION}"]],
107-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
108-
)
108+
sh'''#!/usr/bin/env bash
109+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
110+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
111+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
112+
'''
109113
}
110114
dir("${WORKDIR}") {
111115
withCredentials([
@@ -228,11 +232,11 @@ pipeline {
228232
checkout scm
229233
}
230234
dir("${YETUSDIR}") {
231-
checkout([
232-
$class : 'GitSCM',
233-
branches : [[name: "${YETUS_VERSION}"]],
234-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
235-
)
235+
sh'''#!/usr/bin/env bash
236+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
237+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
238+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
239+
'''
236240
}
237241
dir("${WORKDIR}") {
238242
withCredentials([
@@ -365,11 +369,11 @@ pipeline {
365369
checkout scm
366370
}
367371
dir("${YETUSDIR}") {
368-
checkout([
369-
$class : 'GitSCM',
370-
branches : [[name: "${YETUS_VERSION}"]],
371-
userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
372-
)
372+
sh'''#!/usr/bin/env bash
373+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
374+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
375+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
376+
'''
373377
}
374378
dir("${WORKDIR}") {
375379
withCredentials([
@@ -471,6 +475,143 @@ pipeline {
471475
}
472476
}
473477
}
478+
stage ('yetus jdk17 hadoop3 checks') {
479+
agent {
480+
node {
481+
label 'hbase'
482+
}
483+
}
484+
environment {
485+
// customized per parallel stage
486+
PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
487+
SET_JAVA_HOME = '/usr/lib/jvm/java-17'
488+
HADOOP_PROFILE = '3.0'
489+
WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}"
490+
// identical for all parallel stages
491+
WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
492+
YETUSDIR = "${WORKDIR}/${YETUS_REL}"
493+
SOURCEDIR = "${WORKDIR}/${SRC_REL}"
494+
PATCHDIR = "${WORKDIR}/${PATCH_REL}"
495+
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
496+
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
497+
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
498+
SKIP_ERRORPRONE = true
499+
}
500+
when {
501+
// this will return true if the pipeline is building a change request, such as a GitHub pull request.
502+
changeRequest()
503+
}
504+
steps {
505+
dir("${SOURCEDIR}") {
506+
checkout scm
507+
}
508+
dir("${YETUSDIR}") {
509+
sh'''#!/usr/bin/env bash
510+
wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
511+
tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
512+
rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
513+
'''
514+
}
515+
dir("${WORKDIR}") {
516+
withCredentials([
517+
usernamePassword(
518+
credentialsId: 'apache-hbase-at-github.zerozr99.workers.dev',
519+
passwordVariable: 'GITHUB_PASSWORD',
520+
usernameVariable: 'GITHUB_USER'
521+
)]) {
522+
script {
523+
def ret = sh(
524+
label: 'test-patch',
525+
returnStatus: true,
526+
script: '''#!/bin/bash -e
527+
hostname -a ; pwd ; ls -la
528+
printenv 2>&1 | sort
529+
echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
530+
"${YETUS_DRIVER}"
531+
'''
532+
)
533+
if (ret != 0) {
534+
// mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
535+
// test output. See HBASE-26339 for more details.
536+
currentBuild.result = 'UNSTABLE'
537+
}
538+
}
539+
}
540+
}
541+
}
542+
post {
543+
always {
544+
junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
545+
allowEmptyResults: true, skipPublishingChecks: true
546+
sh label: 'zip surefire reports', script: '''#!/bin/bash -e
547+
if [ -d "${PATCHDIR}/archiver" ]; then
548+
count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
549+
if [[ 0 -ne ${count} ]]; then
550+
echo "zipping ${count} archived files"
551+
zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
552+
else
553+
echo "No archived files, skipping compressing."
554+
fi
555+
else
556+
echo "No archiver directory, skipping compressing."
557+
fi
558+
'''
559+
sshPublisher(publishers: [
560+
sshPublisherDesc(configName: 'Nightlies',
561+
transfers: [
562+
sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
563+
sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
564+
)
565+
]
566+
)
567+
])
568+
// remove the big test logs zip file, store the nightlies url in test_logs.txt
569+
sh '''#!/bin/bash -e
570+
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
571+
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
572+
rm -rf "${PATCHDIR}/test_logs.zip"
573+
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
574+
else
575+
echo "No test_logs.zip, skipping"
576+
fi
577+
'''
578+
// Has to be relative to WORKSPACE.
579+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
580+
archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
581+
publishHTML target: [
582+
allowMissing: true,
583+
keepAll: true,
584+
alwaysLinkToLastBuild: true,
585+
// Has to be relative to WORKSPACE
586+
reportDir: "${WORKDIR_REL}/${PATCH_REL}",
587+
reportFiles: 'report.html',
588+
reportName: 'PR JDK17 Hadoop3 Check Report'
589+
]
590+
}
591+
// Jenkins pipeline jobs fill slaves on PRs without this :(
592+
cleanup() {
593+
script {
594+
sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
595+
# See YETUS-764
596+
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
597+
echo "test-patch process appears to still be running: killing"
598+
kill `cat "${PATCHDIR}/pidfile.txt"` || true
599+
sleep 10
600+
fi
601+
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
602+
echo "test-patch container appears to still be running: killing"
603+
docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
604+
fi
605+
# See HADOOP-13951
606+
chmod -R u+rxw "${WORKSPACE}"
607+
'''
608+
dir ("${WORKDIR}") {
609+
deleteDir()
610+
}
611+
}
612+
}
613+
}
614+
}
474615
}
475616
}
476617
}

‎dev-support/blanks-eol-ignore.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
##
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
##
18+
.*/generated/.*
19+
# we have generated code for other languages in hbase-examples
20+
.*/gen-cpp/.*
21+
.*/gen-perl/.*
22+
.*/gen-php/.*
23+
.*/gen-py/.*
24+
.*/gen-rb/.*

‎dev-support/blanks-tabs-ignore.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
##
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
##
18+
.*/generated/.*
19+
# we have generated code for other languages in hbase-examples
20+
.*/gen-cpp/.*
21+
.*/gen-perl/.*
22+
.*/gen-php/.*
23+
.*/gen-py/.*
24+
.*/gen-rb/.*
25+
# we have tabs in asciidoc, not sure whether it is OK to replace them with spaces
26+
src/main/asciidoc/.*
27+
# perl officially suggests use tab instead of space for indentation
28+
.*/*.pl

0 commit comments

Comments
 (0)