HADOOP-16748. Migrate to Python 3 and upgrade Yetus to 0.13.0

dev-support/Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='6ab19e71eaf3234863424c6f684b34c1d3dcc0ce'
+        YETUS_VERSION='rel/0.13.0'
     }
 
     parameters {
@@ -60,7 +60,7 @@ pipeline {
         stage ('precommit-run') {
             steps {
                 withCredentials(
-                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.zerozr99.workers.dev',
+                    [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
                                   passwordVariable: 'GITHUB_TOKEN',
                                   usernameVariable: 'GITHUB_USER'),
                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
@@ -133,9 +133,6 @@ pipeline {
                         # plugins to enable
                         YETUS_ARGS+=("--plugins=all")
 
-                        # use Hadoop's bundled shelldocs
-                        YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
-
                         # don't let these tests cause -1s because we aren't really paying that
                         # much attention to them
                         YETUS_ARGS+=("--tests-filter=checkstyle")
@@ -152,9 +149,6 @@ pipeline {
                         # help keep the ASF boxes clean
                         YETUS_ARGS+=("--sentinel")
 
-                        # use emoji vote so it is easier to find the broken line
-                        YETUS_ARGS+=("--github-use-emoji-vote")
-
                         # test with Java 8 and 11
                         YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
                         YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
@@ -174,6 +168,19 @@ pipeline {
     post {
         always {
           script {
+            // Publish status if it was missed (YETUS-1059)
+            withCredentials(
+                [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER')]) {
+                sh '''#!/usr/bin/env bash
+                    YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
+                    YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+                    TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
+                    /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
+                    '''
+            }
+
             // Yetus output
             archiveArtifacts "${env.PATCHDIR}/**"
             // Publish the HTML report so that it can be looked at

dev-support/bin/checkcompatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -30,33 +30,16 @@
 import shutil
 import subprocess
 import sys
-import urllib2
-try:
-  import argparse
-except ImportError:
-  sys.stderr.write("Please install argparse, e.g. via `pip install argparse`.")
-  sys.exit(2)
+import urllib.request
+import argparse
 
 # Various relative paths
 REPO_DIR = os.getcwd()
 
 def check_output(*popenargs, **kwargs):
-  r"""Run command with arguments and return its output as a byte string.
-  Backported from Python 2.7 as it's implemented as pure python on stdlib.
-  >>> check_output(['/usr/bin/python', '--version'])
-  Python 2.6.2
-  """
-  process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
-  output, _ = process.communicate()
-  retcode = process.poll()
-  if retcode:
-    cmd = kwargs.get("args")
-    if cmd is None:
-      cmd = popenargs[0]
-    error = subprocess.CalledProcessError(retcode, cmd)
-    error.output = output
-    raise error
-  return output
+  """ Run command with arguments and return its output as a string. """
+  return subprocess.check_output(*popenargs, **kwargs, encoding='utf-8')
+
 
 def get_repo_dir():
   """ Return the path to the top of the repo. """
@@ -139,7 +122,7 @@ def checkout_java_acc(force):
   url = "https://github.com/lvc/japi-compliance-checker/archive/1.8.tar.gz"
   scratch_dir = get_scratch_dir()
   path = os.path.join(scratch_dir, os.path.basename(url))
-  jacc = urllib2.urlopen(url)
+  jacc = urllib.request.urlopen(url)
   with open(path, 'wb') as w:
     w.write(jacc.read())
 
@@ -194,7 +177,7 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
     annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
     with file(annotations_path, "w") as f:
       for ann in annotations:
-        print >>f, ann
+        print(ann, file=f)
     args += ["-annotations-list", annotations_path]
 
   subprocess.check_call(args)
@@ -264,8 +247,8 @@ def main():
   parser.add_argument("--skip-build",
                       action="store_true",
                       help="Skip building the projects.")
-  parser.add_argument("src_rev", nargs=1, help="Source revision.")
-  parser.add_argument("dst_rev", nargs="?", default="HEAD",
+  parser.add_argument("src_rev", nargs=1, type=str, help="Source revision.")
+  parser.add_argument("dst_rev", nargs="?", type=str, default="HEAD",
                       help="Destination revision. " +
                       "If not specified, will use HEAD.")
 

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.10.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.13.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 

dev-support/determine-flaky-tests-hadoop.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -35,22 +35,8 @@
 # at the failed test for the specific run is necessary.
 #
 import sys
-import platform
-sysversion = sys.hexversion
-onward30 = False
-if sysversion < 0x020600F0:
-  sys.exit("Minimum supported python version is 2.6, the current version is " +
-      "Python" + platform.python_version())
-
-if sysversion == 0x030000F0:
-  sys.exit("There is a known bug with Python" + platform.python_version() +
-      ", please try a different version");
-
-if sysversion < 0x03000000:
-  import urllib2
-else:
-  onward30 = True
-  import urllib.request
+
+import urllib.request
 
 import datetime
 import json as simplejson
@@ -99,19 +85,16 @@ def parse_args():
 
 """ Load data from specified url """
 def load_url_data(url):
-  if onward30:
-    ourl = urllib.request.urlopen(url)
-    codec = ourl.info().get_param('charset')
-    content = ourl.read().decode(codec)
-    data = simplejson.loads(content, strict=False)
-  else:
-    ourl = urllib2.urlopen(url)
-    data = simplejson.load(ourl, strict=False)
+  ourl = urllib.request.urlopen(url)
+  codec = ourl.info().get_param('charset')
+  content = ourl.read().decode(codec)
+  data = simplejson.loads(content, strict=False)
   return data
- 
+
 """ List all builds of the target project. """
 def list_builds(jenkins_url, job_name):
   global summary_mode
+  global error_count
   url = "%(jenkins)s/job/%(job_name)s/api/json?tree=builds[url,result,timestamp]" % dict(
       jenkins=jenkins_url,
       job_name=job_name)
@@ -162,8 +145,8 @@ def find_flaky_tests(jenkins_url, job_name, num_prev_days):
   builds = list_builds(jenkins_url, job_name)
 
   # Select only those in the last N days
-  min_time = int(time.time()) - SECONDS_PER_DAY * num_prev_days
-  builds = [b for b in builds if (int(b['timestamp']) / 1000) > min_time]
+  min_time = time.time() - SECONDS_PER_DAY * num_prev_days
+  builds = [b for b in builds if (float(b['timestamp']) / 1000) > min_time]
 
   # Filter out only those that failed
   failing_build_urls = [(b['url'] , b['timestamp']) for b in builds

dev-support/docker/Dockerfile

@@ -72,10 +72,11 @@ RUN apt-get -q update \
         openjdk-8-jdk \
         pinentry-curses \
         pkg-config \
-        python \
-        python2.7 \
-        python-pkg-resources \
-        python-setuptools \
+        python3 \
+        python3-pip \
+        python3-pkg-resources \
+        python3-setuptools \
+        python3-wheel \
         rsync \
         shellcheck \
         software-properties-common \
@@ -108,18 +109,6 @@ RUN mkdir -p /opt/boost-library \
     && cd /root \
     && rm -rf /opt/boost-library
 
-####
-# Install pip (deprecated from Focal toolchain)
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/pip \
-    && curl -L https://bootstrap.pypa.io/2.7/get-pip.py > get-pip.py \
-    && mv get-pip.py /opt/pip \
-    && cd /opt/pip \
-    && python2.7 get-pip.py "pip < 21.0" \
-    && cd /root \
-    && rm -rf /opt/pip
-
 ######
 # Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
 ######
@@ -139,19 +128,9 @@ ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
 ####
-# Install pylint at fixed version (2.0.0 removed python2 support)
-# https://github.com/PyCQA/pylint/issues/2294
-####
-RUN pip2 install \
-    astroid==1.6.6 \
-    isort==4.3.21 \
-    configparser==4.0.2 \
-    pylint==1.9.2
-
-####
-# Install dateutil.parser
+# Install pylint and python-dateutil
 ####
-RUN pip2 install python-dateutil==2.7.3
+RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
 
 ####
 # Install bower

dev-support/docker/Dockerfile_aarch64

@@ -76,10 +76,11 @@ RUN apt-get -q update \
         openjdk-8-jdk \
         pinentry-curses \
         pkg-config \
-        python \
-        python2.7 \
-        python-pkg-resources \
-        python-setuptools \
+        python3 \
+        python3-pip \
+        python3-pkg-resources \
+        python3-setuptools \
+        python3-wheel \
         rsync \
         shellcheck \
         software-properties-common \
@@ -112,18 +113,6 @@ RUN mkdir -p /opt/boost-library \
     && cd /root \
     && rm -rf /opt/boost-library
 
-####
-# Install pip (deprecated from Focal toolchain)
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/pip \
-    && curl -L https://bootstrap.pypa.io/2.7/get-pip.py > get-pip.py \
-    && mv get-pip.py /opt/pip \
-    && cd /opt/pip \
-    && python2.7 get-pip.py "pip < 21.0" \
-    && cd /root \
-    && rm -rf /opt/pip
-
 ######
 # Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
 ######
@@ -143,19 +132,9 @@ ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
 ####
-# Install pylint at fixed version (2.0.0 removed python2 support)
-# https://github.com/PyCQA/pylint/issues/2294
-####
-RUN pip2 install \
-    astroid==1.6.6 \
-    isort==4.3.21 \
-    configparser==4.0.2 \
-    pylint==1.9.2
-
-####
-# Install dateutil.parser
+# Install pylint and python-dateutil
 ####
-RUN pip2 install python-dateutil==2.7.3
+RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
 
 ####
 # Install bower