TEZ-4292: Upgrade to Yetus 0.13

.yetus/personality.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+## @description  Globals specific to this personality
+## @audience     private
+## @stability    evolving
+function personality_globals
+{
+  # shellcheck disable=SC2034
+  BUILDTOOL=maven
+  #shellcheck disable=SC2034
+  PATCH_BRANCH_DEFAULT=master
+  #shellcheck disable=SC2034
+  PROJECT="tez"
+  #shellcheck disable=SC2034
+  JIRA_ISSUE_RE='^TEZ-[0-9]+$'
+  #shellcheck disable=SC2034
+  GITHUB_REPO_DEFAULT="apache/tez"
+  #shellcheck disable=SC2034
+  PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"
+}

Jenkinsfile

@@ -35,14 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.12.0'
-
-    }
-
-    parameters {
-        string(name: 'JIRA_ISSUE_KEY',
-               defaultValue: '',
-               description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
+        YETUS_VERSION='6ab19e71eaf3234863424c6f684b34c1d3dcc0ce'
     }
 
     stages {
@@ -52,7 +45,7 @@ pipeline {
                     checkout([
                         $class: 'GitSCM',
                         branches: [[name: "${env.YETUS_VERSION}"]],
-                        userRemoteConfigs: [[ url: 'https://github.com/jteagles/yetus']]]
+                        userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]]
                     )
                 }
             }
@@ -62,7 +55,7 @@ pipeline {
             steps {
                 withCredentials(
                     [usernamePassword(credentialsId: 'apache-tez-at-github.zerozr99.workers.dev',
-                                        passwordVariable: 'GITHUB_PASSWD',
+                                        passwordVariable: 'GITHUB_TOKEN',
                                         usernameVariable: 'GITHUB_USER'),
                                         usernamePassword(credentialsId: 'tez-ci',
                                         passwordVariable: 'JIRA_PASSWORD',
@@ -79,39 +72,30 @@ pipeline {
                         fi
                         mkdir -p "${WORKSPACE}/${PATCHDIR}"
 
-                        # if given a JIRA issue, process it. If CHANGE_URL is set
-                        # (e.g., Github Branch Source plugin), process it.
-                        # otherwise exit, because we don't want Hadoop to do a
-                        # full build.  We wouldn't normally do this check for smaller
-                        # projects. :)
-                        if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
-                            YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
-                        elif [[ -z "${CHANGE_URL}" ]]; then
-                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
-                            exit 0
-                        fi
-
                         YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
 
                         # where the source is located
                         YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
 
                         # our project defaults come from a personality file
                         YETUS_ARGS+=("--project=tez")
+                        YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/.yetus/personality.sh")
 
                         # lots of different output formats
                         YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
                         YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 
                         # enable writing back to Github
-                        YETUS_ARGS+=(--github-user="${GITHUB_USER}")
-                        YETUS_ARGS+=(--github-password="${GITHUB_PASSWD}")
+                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
 
                         # enable writing back to ASF JIRA
                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
                         YETUS_ARGS+=(--jira-user="${JIRA_USER}")
 
+                        # enable everything
+                        YETUS_ARGS+=("--plugins=all")
+
                         # auto-kill any surefire stragglers during unit test runs
                         YETUS_ARGS+=("--reapermode=kill")
 
@@ -131,12 +115,6 @@ pipeline {
                         # (needs to match the archive bits below)
                         YETUS_ARGS+=("--build-url-artifacts=artifact/out")
 
-                        # plugins to enable
-                        YETUS_ARGS+=("--plugins=all")
-
-                        # use Hadoop's bundled shelldocs
-                        YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
-
                         # don't let these tests cause -1s because we aren't really paying that
                         # much attention to them
                         YETUS_ARGS+=("--tests-filter=checkstyle")
@@ -147,17 +125,13 @@ pipeline {
                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
                         YETUS_ARGS+=("--mvn-custom-repos")
 
-                        # effectively treat dev-suport as a custom maven module
-                        YETUS_ARGS+=("--skip-dirs=dev-support")
-
                         # help keep the ASF boxes clean
                         YETUS_ARGS+=("--sentinel")
 
                         # test with Java 8 and 11
                         YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
                         YETUS_ARGS+=("--multijdktests=compile")
-                        YETUS_ARGS+=("--debug")
 
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                         '''
@@ -170,6 +144,27 @@ pipeline {
     post {
         always {
           script {
+
+            // Publish status if it was missed
+            withCredentials([usernamePassword(credentialsId: 'apache-tez-at-github.zerozr99.workers.dev',
+                             passwordVariable: 'GITHUB_TOKEN',
+                             usernameVariable: 'GITHUB_USER')]) {
+                    sh '''#!/usr/bin/env bash
+
+                    # enable writing back to Github
+                    YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
+
+                    YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
+
+                    # run test-patch from the source tree specified up above
+                    TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
+
+                    # execute! (we are using bash instead of the
+                    # bin in case the perms get messed up)
+                    /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" || true
+                    '''
+            }
+
             // Yetus output
             archiveArtifacts "${env.PATCHDIR}/**"
             // Publish the HTML report so that it can be looked at