1+ // Licensed to the Apache Software Foundation (ASF) under one
2+ // or more contributor license agreements. See the NOTICE file
3+ // distributed with this work for additional information
4+ // regarding copyright ownership. The ASF licenses this file
5+ // to you under the Apache License, Version 2.0 (the
6+ // "License"); you may not use this file except in compliance
7+ // with the License. You may obtain a copy of the License at
8+ //
9+ // http://www.apache.org/licenses/LICENSE-2.0
10+ //
11+ // Unless required by applicable law or agreed to in writing,
12+ // software distributed under the License is distributed on an
13+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+ // KIND, either express or implied. See the License for the
15+ // specific language governing permissions and limitations
16+ // under the License.
17+
18+ pipeline {
19+
20+ agent {
21+ label ' Hadoop'
22+ }
23+
24+ options {
25+ buildDiscarder(logRotator(numToKeepStr : ' 5'
26+ timeout (time : 20 , unit : ' HOURS'
27+ timestamps()
28+ checkoutToSubdirectory(' src'
29+ }
30+
31+ environment {
32+ SOURCEDIR = ' src'
33+ // will also need to change notification section below
34+ PATCHDIR = ' out'
35+ DOCKERFILE = " ${ SOURCEDIR} "
36+ YETUS = ' yetus'
37+ // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
38+ YETUS_VERSION = ' rel/0.12.0'
39+
40+ }
41+
42+ parameters {
43+ string(name : ' JIRA_ISSUE_KEY'
44+ defaultValue : ' '
45+ description : ' The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234'
46+ }
47+
48+ stages {
49+ stage (' install yetus'
50+ steps {
51+ dir(" ${ WORKSPACE} ${ YETUS} "
52+ checkout([
53+ $class : ' GitSCM'
54+ branches : [[name : " ${ env.YETUS_VERSION} "
55+ userRemoteConfigs : [[ url : ' https://github.com/apache/yetus'
56+ )
57+ }
58+ }
59+ }
60+
61+ stage (' precommit-run'
62+ steps {
63+ withCredentials(
64+ [usernamePassword(credentialsId : ' apache-hadoop-at-github.zerozr99.workers.dev'
65+ passwordVariable : ' GITHUB_TOKEN'
66+ usernameVariable : ' GITHUB_USER'
67+ usernamePassword(credentialsId : ' tez-ci'
68+ passwordVariable : ' JIRA_PASSWORD'
69+ usernameVariable : ' JIRA_USER'
70+ sh ''' #!/usr/bin/env bash
71+
72+ set -e
73+
74+ TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
75+
76+ # this must be clean for every run
77+ if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
78+ rm -rf "${WORKSPACE}/${PATCHDIR}"
79+ fi
80+ mkdir -p "${WORKSPACE}/${PATCHDIR}"
81+
82+ # if given a JIRA issue, process it. If CHANGE_URL is set
83+ # (e.g., Github Branch Source plugin), process it.
84+ # otherwise exit, because we don't want Hadoop to do a
85+ # full build. We wouldn't normally do this check for smaller
86+ # projects. :)
87+ if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
88+ YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
89+ elif [[ -z "${CHANGE_URL}" ]]; then
90+ echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
91+ exit 0
92+ fi
93+
94+ YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
95+
96+ # where the source is located
97+ YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
98+
99+ # our project defaults come from a personality file
100+ YETUS_ARGS+=("--project=tez")
101+
102+ # lots of different output formats
103+ YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
104+ YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
105+ YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
106+
107+ # enable writing back to Github
108+ YETUS_ARGS+=(--github-user="${GITHUB_USER}")
109+ YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
110+
111+ # enable writing back to ASF JIRA
112+ YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
113+ YETUS_ARGS+=(--jira-user="${JIRA_USER}")
114+
115+ # auto-kill any surefire stragglers during unit test runs
116+ YETUS_ARGS+=("--reapermode=kill")
117+
118+ # set relatively high limits for ASF machines
119+ # changing these to higher values may cause problems
120+ # with other jobs on systemd-enabled machines
121+ YETUS_ARGS+=("--proclimit=5500")
122+ YETUS_ARGS+=("--dockermemlimit=20g")
123+
124+ # -1 findbugs issues that show up prior to the patch being applied
125+ # YETUS_ARGS+=("--findbugs-strict-precheck")
126+
127+ # rsync these files back into the archive dir
128+ YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
129+
130+ # URL for user-side presentation in reports and such to our artifacts
131+ # (needs to match the archive bits below)
132+ YETUS_ARGS+=("--build-url-artifacts=artifact/out")
133+
134+ # plugins to enable
135+ YETUS_ARGS+=("--plugins=all")
136+
137+ # use Hadoop's bundled shelldocs
138+ YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
139+
140+ # don't let these tests cause -1s because we aren't really paying that
141+ # much attention to them
142+ YETUS_ARGS+=("--tests-filter=checkstyle")
143+
144+ # run in docker mode and specifically point to our
145+ # Dockerfile since we don't want to use the auto-pulled version.
146+ YETUS_ARGS+=("--docker")
147+ YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
148+ YETUS_ARGS+=("--mvn-custom-repos")
149+
150+ # effectively treat dev-suport as a custom maven module
151+ YETUS_ARGS+=("--skip-dirs=dev-support")
152+
153+ # help keep the ASF boxes clean
154+ YETUS_ARGS+=("--sentinel")
155+
156+ # use emoji vote so it is easier to find the broken line
157+ YETUS_ARGS+=("--github-use-emoji-vote")
158+
159+ # test with Java 8 and 11
160+ YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
161+ YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
162+ YETUS_ARGS+=("--multijdktests=compile")
163+ YETUS_ARGS+=("--debug")
164+
165+ "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
166+ '''
167+ }
168+ }
169+ }
170+
171+ }
172+
173+ post {
174+ always {
175+ script {
176+ // Yetus output
177+ archiveArtifacts " ${ env.PATCHDIR} "
178+ // Publish the HTML report so that it can be looked at
179+ // Has to be relative to WORKSPACE.
180+ publishHTML (target : [
181+ allowMissing : true ,
182+ keepAll : true ,
183+ alwaysLinkToLastBuild : true ,
184+ // Has to be relative to WORKSPACE
185+ reportDir : " ${ env.PATCHDIR} "
186+ reportFiles : ' report.html'
187+ reportName : ' Yetus Report'
188+ ])
189+ // Publish JUnit results
190+ try {
191+ junit " ${ env.SOURCEDIR} "
192+ } catch (e) {
193+ echo ' junit processing: ' + e. toString()
194+ }
195+ }
196+ }
197+
198+ // Jenkins pipeline jobs fill slaves on PRs without this :(
199+ cleanup() {
200+ script {
201+ sh '''
202+ # See YETUS-764
203+ if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
204+ echo "test-patch process appears to still be running: killing"
205+ kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
206+ sleep 10
207+ fi
208+ if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
209+ echo "test-patch container appears to still be running: killing"
210+ docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
211+ fi
212+ # See HADOOP-13951
213+ chmod -R u+rxw "${WORKSPACE}"
214+ '''
215+ deleteDir()
216+ }
217+ }
218+ }
219+ }
0 commit comments