initial support for --close-pr (REVIEW)

easybuild/main.py

@@ -60,7 +60,7 @@
 from easybuild.tools.docs import list_software
 from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
 from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token
-from easybuild.tools.github import new_pr, merge_pr, update_pr
+from easybuild.tools.github import close_pr, new_pr, merge_pr, update_pr
 from easybuild.tools.hooks import START, END, load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import parse_external_modules_metadata, process_software_build_specs, use_color
@@ -284,6 +284,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     elif options.install_github_token:
         install_github_token(options.github_user, silent=build_option('silent'))
 
+    elif options.close_pr:
+        close_pr(options.close_pr, reasons=options.close_pr_msg)
+
     elif options.merge_pr:
         merge_pr(options.merge_pr)
 
@@ -303,6 +306,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         options.install_github_token,
         options.list_installed_software,
         options.list_software,
+        options.close_pr,
         options.merge_pr,
         options.review_pr,
         options.terse,

easybuild/tools/github.py

@@ -41,13 +41,16 @@
 import tempfile
 import time
 import urllib2
+from datetime import datetime, timedelta
 from distutils.version import LooseVersion
 from vsc.utils import fancylogger
 from vsc.utils.missing import nub
 
+from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR
 from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, process_easyconfig
 from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION
 from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION
+from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import apply_patch, copy_dir, det_patched_files, download_file, extract_file
@@ -96,6 +99,9 @@
 KEYRING_GITHUB_TOKEN = 'github_token'
 URL_SEPARATOR = '/'
 
+VALID_CLOSE_PR_REASONS = {'archived': 'uses an archived toolchain',
+                          'inactive': 'no activity for > 6 months',
+                          'obsolete': 'obsoleted by more recent PRs'}
 
 class Githubfs(object):
     """This class implements some higher level functionality on top of the Github api"""
@@ -369,12 +375,8 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None):
         mkdir(path, parents=True)
 
     _log.debug("Fetching easyconfigs from PR #%s into %s" % (pr, path))
-    pr_url = lambda g: g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr]
 
-    status, pr_data = github_api_get_request(pr_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
-                             pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data)
+    status, pr_data, pr_url = fetch_pr_data(pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, github_user)
 
     # if PR is open and mergable, download develop and patch
     stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN
@@ -970,6 +972,139 @@ def not_eligible(msg):
     return res
 
 
+def reasons_for_closing(pr_data):
+    """
+    Look for valid reasons to close PR by comparing with existing easyconfigs.
+    """
+    print_msg("No reason or message specified, looking for possible reasons\n")
+
+    if pr_data['status_last_commit']:
+        print_msg("Status of last commit is %s\n" % pr_data['status_last_commit'].upper(), prefix=False)
+
+    if pr_data['issue_comments']:
+        last_comment = pr_data['issue_comments'][-1]
+        print_msg("Last comment on %s, by %s, was:\n\n%s" %
+                  (last_comment['updated_at'].replace('T', ' at ')[:-1], last_comment['user']['login'],
+                   last_comment['body']), prefix=False)
+
+    if pr_data['reviews']:
+        last_review = pr_data['reviews'][-1]
+        print_msg("Last reviewed on %s by %s, state %s\n\n%s" %
+                  (last_review['submitted_at'].replace('T', ' at '), last_review['user']['login'],
+                   last_review['state'], last_review['body']), prefix=False)
+
+    possible_reasons = []
+
+    print_msg("No activity since %s" % pr_data['updated_at'].replace('T', ' at ')[:-1], prefix=False)
+
+    # check if PR is inactive for more than 6 months
+    last_updated = datetime.strptime(pr_data['updated_at'], "%Y-%m-%dT%H:%M:%SZ")
+    if datetime.now() - last_updated > timedelta(days=180):
+        possible_reasons.append('inactive')
+
+    robot_paths = build_option('robot_path')
+
+    pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')]
+
+    obsoleted = []
+    uses_archived_tc = []
+    for pr_file in pr_files:
+        pr_ec = EasyConfigParser(pr_file).get_config_dict()
+        pr_tc = '%s-%s' % (pr_ec['toolchain']['name'], pr_ec['toolchain']['version'])
+        print_msg("* %s-%s" % (pr_ec['name'], pr_ec['version']), prefix=False)
+        for robot_path in robot_paths:
+            # check if PR easyconfig uses an archived toolchain
+            path = os.path.join(robot_path, EASYCONFIGS_ARCHIVE_DIR, pr_tc[0].lower(), pr_tc.split('-')[0])
+            for (dirpath, _, filenames) in os.walk(path):
+                for fn in filenames:
+                    if fn.endswith('.eb'):
+                        ec = EasyConfigParser(os.path.join(dirpath, fn)).get_config_dict()
+                        if ec.get('easyblock') == 'Toolchain':
+                            if 'versionsuffix' in ec:
+                                archived_tc = '%s-%s%s' % (ec['name'], ec['version'], ec.get('versionsuffix'))
+                            else:
+                                archived_tc = '%s-%s' % (ec['name'], ec['version'])
+                            if pr_tc == archived_tc:
+                                print_msg(" - uses archived toolchain %s" % pr_tc, prefix=False)
+                                uses_archived_tc.append(pr_ec)
+
+            # check if there is a newer version of PR easyconfig
+            newer_versions = set()
+            for (dirpath, _, filenames) in os.walk(os.path.join(robot_path, pr_ec['name'].lower()[0], pr_ec['name'])):
+                for fn in filenames:
+                    if fn.endswith('.eb'):
+                        ec = EasyConfigParser(os.path.join(dirpath, fn)).get_config_dict()
+                        if LooseVersion(ec['version']) > LooseVersion(pr_ec['version']):
+                            newer_versions.add(ec['version'])
+
+            if newer_versions:
+                print_msg(" - found newer versions %s" % ", ".join(sorted(newer_versions)), prefix=False)
+                obsoleted.append(pr_ec)
+
+    if uses_archived_tc:
+        possible_reasons.append('archived')
+
+    if any([ec['name'] in pr_data['title'] for ec in obsoleted]):
+        possible_reasons.append('obsolete')
+
+    return possible_reasons
+
+
+def close_pr(pr, reasons):
+    """
+    Close specified pull request
+    """
+    github_user = build_option('github_user')
+    if github_user is None:
+        raise EasyBuildError("GitHub user must be specified to use --close-pr")
+
+    pr_target_account = build_option('pr_target_account')
+    pr_target_repo = build_option('pr_target_repo')
+
+    status, pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True)
+
+    if pr_data['state'] == GITHUB_STATE_CLOSED:
+        raise EasyBuildError("PR #%d from %s/%s is already closed.", pr, pr_target_account, pr_target_repo)
+
+    pr_owner = pr_data['user']['login']
+    msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_owner)
+    msg += "you are using GitHub account '%s'\n" % github_user
+    msg += "\nPR Title: \"%s\"\n" % pr_data['title']
+    print_msg(msg, prefix=False)
+
+    dry_run = build_option('dry_run') or build_option('extended_dry_run')
+
+    if not reasons:
+        possible_reasons = reasons_for_closing(pr_data)
+
+        if not possible_reasons:
+            raise EasyBuildError("No reason specified and none found from PR data, "
+                                 "please use --close-pr-reasons or --close-pr-msg")
+        else:
+            reasons = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in possible_reasons])
+            print_msg("\nNo reason specified but found possible reasons: %s.\n" % reasons, prefix=False)
+
+    msg = "@%s, this PR is being closed for the following reason(s): %s.\n" % (pr_data['user']['login'], reasons)
+    msg += "Please don't hesitate to reopen this PR or add a comment if you feel this contribution is still relevant.\n"
+    msg += "For more information on our policy w.r.t. closing PRs, see "
+    msg += "https://easybuild.readthedocs.io/en/latest/Contributing.html"
+    msg += "#why-a-pull-request-may-be-closed-by-a-maintainer"
+    post_comment_in_issue(pr, msg, account=pr_target_account, repo=pr_target_repo, github_user=github_user)
+
+    if dry_run:
+        print_msg("[DRY RUN] Closed %s/%s pull request #%s" % (pr_target_account, pr_target_repo, pr), prefix=False)
+    else:
+        github_token = fetch_github_token(github_user)
+        if github_token is None:
+            raise EasyBuildError("GitHub token for user '%s' must be available to use --close-pr", github_user)
+        g = RestClient(GITHUB_API_URL, username=github_user, token=github_token)
+        pull_url = g.repos[pr_target_account][pr_target_repo].pulls[pr]
+        body = {'state': 'closed'}
+        status, data = pull_url.post(body=body)
+        if not status == HTTP_STATUS_OK:
+            raise EasyBuildError("Failed to close PR #%s; status %s, data: %s", pr, status, data)
+
+
 def merge_pr(pr):
     """
     Merge specified pull request
@@ -981,43 +1116,14 @@ def merge_pr(pr):
     pr_target_account = build_option('pr_target_account')
     pr_target_repo = build_option('pr_target_repo')
 
-    pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr]
-    status, pr_data = github_api_get_request(pr_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
-                             pr, pr_target_account, pr_target_repo, status, pr_data)
+    status, pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True)
 
     msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login'])
     msg += "you are using GitHub account '%s'\n" % github_user
     print_msg(msg, prefix=False)
     if pr_data['user']['login'] == github_user:
         raise EasyBuildError("Please do not merge your own PRs!")
 
-    # also fetch status of last commit
-    pr_head_sha = pr_data['head']['sha']
-    status_url = lambda g: g.repos[pr_target_account][pr_target_repo].commits[pr_head_sha].status
-    status, status_data = github_api_get_request(status_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get status of last commit for PR #%d from %s/%s (status: %d %s)",
-                             pr, pr_target_account, pr_target_repo, status, status_data)
-    pr_data['status_last_commit'] = status_data['state']
-
-    # also fetch comments
-    comments_url = lambda g: g.repos[pr_target_account][pr_target_repo].issues[pr].comments
-    status, comments_data = github_api_get_request(comments_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)",
-                             pr, pr_target_account, pr_target_repo, status, comments_data)
-    pr_data['issue_comments'] = comments_data
-
-    # also fetch reviews
-    reviews_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr].reviews
-    status, reviews_data = github_api_get_request(reviews_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
-                             pr, pr_target_account, pr_target_repo, status, reviews_data)
-    pr_data['reviews'] = reviews_data
-
     force = build_option('force')
     dry_run = build_option('dry_run') or build_option('extended_dry_run')
 
@@ -1175,11 +1281,7 @@ def update_pr(pr, paths, ecs, commit_msg=None):
     pr_target_account = build_option('pr_target_account')
     pr_target_repo = build_option('pr_target_repo')
 
-    pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr]
-    status, pr_data = github_api_get_request(pr_url, github_user)
-    if status != HTTP_STATUS_OK:
-        raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
-                             pr, pr_target_account, pr_target_repo, status, pr_data)
+    status, pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user)
 
     # branch that corresponds with PR is supplied in form <account>:<branch_label>
     account = pr_data['head']['label'].split(':')[0]
@@ -1517,3 +1619,40 @@ def find_easybuild_easyconfig(github_user=None):
 
     eb_file = os.path.join(eb_parent_path, fn)
     return eb_file
+
+
+def fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=False):
+    """Fetch PR data from GitHub"""
+    pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr]
+    status, pr_data = github_api_get_request(pr_url, github_user)
+    if status != HTTP_STATUS_OK:
+        raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)",
+                             pr, pr_target_account, pr_target_repo, status, pr_data)
+
+    if full:
+        # also fetch status of last commit
+        pr_head_sha = pr_data['head']['sha']
+        status_url = lambda g: g.repos[pr_target_account][pr_target_repo].commits[pr_head_sha].status
+        status, status_data = github_api_get_request(status_url, github_user)
+        if status != HTTP_STATUS_OK:
+            raise EasyBuildError("Failed to get status of last commit for PR #%d from %s/%s (status: %d %s)",
+                                 pr, pr_target_account, pr_target_repo, status, status_data)
+        pr_data['status_last_commit'] = status_data['state']
+
+        # also fetch comments
+        comments_url = lambda g: g.repos[pr_target_account][pr_target_repo].issues[pr].comments
+        status, comments_data = github_api_get_request(comments_url, github_user)
+        if status != HTTP_STATUS_OK:
+            raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)",
+                                 pr, pr_target_account, pr_target_repo, status, comments_data)
+        pr_data['issue_comments'] = comments_data
+
+        # also fetch reviews
+        reviews_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr].reviews
+        status, reviews_data = github_api_get_request(reviews_url, github_user)
+        if status != HTTP_STATUS_OK:
+            raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)",
+                                 pr, pr_target_account, pr_target_repo, status, reviews_data)
+        pr_data['reviews'] = reviews_data
+
+    return status, pr_data, pr_url

easybuild/tools/options.py

@@ -72,6 +72,7 @@
 from easybuild.tools.environment import restore_env, unset_env_vars
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, mkdir
 from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, HAVE_GITHUB_API, HAVE_KEYRING
+from easybuild.tools.github import VALID_CLOSE_PR_REASONS
 from easybuild.tools.github import fetch_github_token
 from easybuild.tools.hooks import KNOWN_HOOKS
 from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains
@@ -126,7 +127,6 @@ def eb_shell_quote(token):
 DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))]
 DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg')
 
-
 _log = fancylogger.getLogger('options', fname=False)
 
 
@@ -567,6 +567,10 @@ def github_options(self):
             'github-user': ("GitHub username", str, 'store', None),
             'github-org': ("GitHub organization", str, 'store', None),
             'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False),
+            'close-pr': ("Close pull request", int, 'store', None, {'metavar': 'PR#'}),
+            'close-pr-msg': ("Custom close message for pull request closed with --close-pr; ", str, 'store', None),
+            'close-pr-reasons': ("Close reason for pull request closed with --close-pr; "
+                                 "supported values: %s" % ", ".join(VALID_CLOSE_PR_REASONS), str, 'store', None),
             'merge-pr': ("Merge pull request", int, 'store', None, {'metavar': 'PR#'}),
             'new-pr': ("Open a new pull request", None, 'store_true', False),
             'pr-branch-name': ("Branch name to use for new PRs; '<timestamp>_new_pr_<name><version>' if unspecified",
@@ -786,6 +790,10 @@ def postprocess(self):
         if self.options.optarch and not self.options.job:
             self._postprocess_optarch()
 
+        # make sure --close-pr-reasons has a valid format and if so use it to set close-pr-msg
+        if self.options.close_pr_reasons:
+            self._postprocess_close_pr_reasons()
+
         # handle configuration options that affect other configuration options
         self._postprocess_config()
 
@@ -819,6 +827,18 @@ def _postprocess_optarch(self):
             else:
                 self.log.info("Keeping optarch raw: %s", self.options.optarch)
 
+    def _postprocess_close_pr_reasons(self):
+        """Postprocess --close-pr-reasons options"""
+        if self.options.close_pr_msg:
+            raise EasyBuildError("Please either specify predefined reasons with --close-pr-reasons or " +
+                                 "a custom message with--close-pr-msg")
+
+        reasons = self.options.close_pr_reasons.split(',')
+        if any([reason not in VALID_CLOSE_PR_REASONS.keys() for reason in reasons]):
+            raise EasyBuildError("Argument to --close-pr_reasons must be a comma separated list of valid reasons " +
+                                 "among %s" % VALID_CLOSE_PR_REASONS.keys())
+        self.options.close_pr_msg = ", ".join([VALID_CLOSE_PR_REASONS[reason] for reason in reasons])
+
     def _postprocess_include(self):
         """Postprocess --include options."""
         # set up included easyblocks, module naming schemes and toolchains/toolchain components

test/framework/github.py

@@ -114,6 +114,18 @@ def test_read(self):
         except (IOError, OSError):
             pass
 
+    def test_fetch_pr_data(self):
+        """Test fetch_pr_data function."""
+        if self.github_token is None:
+            print "Skipping test_fetch_pr_data, no GitHub token available?"
+            return
+
+        status, pr_data, pr_url = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT)
+
+        self.assertEquals(gh.HTTP_STATUS_OK, status)
+        self.assertEquals(pr_data['number'], 1)
+        self.assertEquals(pr_data['title'], "a pr")
+
     def test_fetch_easyconfigs_from_pr(self):
         """Test fetch_easyconfigs_from_pr function."""
         if self.github_token is None: