diff --git a/easybuild/main.py b/easybuild/main.py index 4de63d1166..85eba47f6f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -58,7 +58,7 @@ from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, write_file from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token -from easybuild.tools.github import new_pr, merge_pr, update_pr +from easybuild.tools.github import list_prs, new_pr, merge_pr, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_up_configuration, use_color @@ -230,6 +230,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) + elif options.list_prs: + print list_prs(options.list_prs) + elif options.merge_pr: merge_pr(options.merge_pr) @@ -249,6 +252,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): options.install_github_token, options.list_installed_software, options.list_software, + options.list_prs, options.merge_pr, options.review_pr, options.terse, diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c6be3e2c7c..03258f2e63 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -84,6 +84,12 @@ GITHUB_EB_MAIN = 'easybuilders' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' GITHUB_FILE_TYPE = u'file' +GITHUB_PR_STATE_OPEN = 'open' +GITHUB_PR_STATES = [GITHUB_PR_STATE_OPEN, 'closed', 'all'] +GITHUB_PR_ORDER_CREATED = 'created' +GITHUB_PR_ORDERS = [GITHUB_PR_ORDER_CREATED, 'updated', 'popularity', 'long-running'] +GITHUB_PR_DIRECTION_DESC = 'desc' +GITHUB_PR_DIRECTIONS = ['asc', GITHUB_PR_DIRECTION_DESC] GITHUB_MAX_PER_PAGE = 100 GITHUB_MERGEABLE_STATE_CLEAN = 'clean' GITHUB_PR = 'pull' @@ -211,7 +217,6 @@ class GithubError(Exception): pass - def github_api_get_request(request_f, github_user=None, token=None, **kwargs): """ Helper method, for performing get requests to GitHub API. @@ -879,7 +884,6 @@ def find_software_name_for_patch(patch_name): :return: name of the software that this patch file belongs to (if found) """ - robot_paths = build_option('robot_path') soft_name = None @@ -991,6 +995,40 @@ def not_eligible(msg): return res +def list_prs(params, per_page=GITHUB_MAX_PER_PAGE): + """ + List pull requests according to specified selection/order parameters + + :param params: 3-tuple with selection parameters for PRs (, , ), + see https://developer.github.com/v3/pulls/#parameters + """ + parameters = { + 'state': params[0], + 'sort': params[1], + 'direction': params[2], + 'per_page': per_page, + } + print_msg("Listing PRs with parameters: %s" % ', '.join(k + '=' + str(parameters[k]) for k in sorted(parameters))) + + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') + + def pr_url(gh): + """Utility function to fetch data for PRs.""" + return gh.repos[pr_target_account][pr_target_repo].pulls + + status, pr_data = github_api_get_request(pr_url, None, **parameters) + if status != HTTP_STATUS_OK: + raise EasyBuildError("Failed to get PR data from %s/%s (parameters: %s, status: %d %s)", + pr_target_account, pr_target_repo, parameters, status, pr_data) + + lines = [] + for pr in pr_data: + lines.append("PR #%s: %s" % (pr['number'], pr['title'])) + + return '\n'.join(lines) + + def merge_pr(pr): """ Merge specified pull request @@ -1002,7 +1040,10 @@ def merge_pr(pr): pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') - pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr] + def pr_url(gh): + """Utility function to fetch data for a specific PR.""" + return gh.repos[pr_target_account][pr_target_repo].pulls[pr] + status, pr_data = github_api_get_request(pr_url, github_user) if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", @@ -1014,25 +1055,35 @@ def merge_pr(pr): if pr_data['user']['login'] == github_user: raise EasyBuildError("Please do not merge your own PRs!") - # also fetch status of last commit pr_head_sha = pr_data['head']['sha'] - status_url = lambda g: g.repos[pr_target_account][pr_target_repo].commits[pr_head_sha].status + + def status_url(gh): + """Utility function to fetch status of specific commit.""" + return gh.repos[pr_target_account][pr_target_repo].commits[pr_head_sha].status + + # also fetch status of last commit status, status_data = github_api_get_request(status_url, github_user) if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get status of last commit for PR #%d from %s/%s (status: %d %s)", pr, pr_target_account, pr_target_repo, status, status_data) pr_data['status_last_commit'] = status_data['state'] + def comments_url(gh): + """Utility function to fetch comments for a specific PR.""" + return gh.repos[pr_target_account][pr_target_repo].issues[pr].comments + # also fetch comments - comments_url = lambda g: g.repos[pr_target_account][pr_target_repo].issues[pr].comments status, comments_data = github_api_get_request(comments_url, github_user) if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get comments for PR #%d from %s/%s (status: %d %s)", pr, pr_target_account, pr_target_repo, status, comments_data) pr_data['issue_comments'] = comments_data + def reviews_url(gh): + """Utility function to fetch reviews for a specific PR.""" + return gh.repos[pr_target_account][pr_target_repo].pulls[pr].reviews + # also fetch reviews - reviews_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr].reviews status, reviews_data = github_api_get_request(reviews_url, github_user) if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get reviews for PR #%d from %s/%s (status: %d %s)", @@ -1042,6 +1093,10 @@ def merge_pr(pr): force = build_option('force') dry_run = build_option('dry_run') or build_option('extended_dry_run') + def merge_url(gh): + """Utility function to fetch merge URL for a specific PR.""" + return gh.repos[pr_target_account][pr_target_repo].pulls[pr].merge + if check_pr_eligible_to_merge(pr_data) or force: print_msg("\nReview %s merging pull request!\n" % ("OK,", "FAILed, yet forcibly")[force], prefix=False) @@ -1055,7 +1110,6 @@ def merge_pr(pr): 'commit_message': pr_data['title'], 'sha': pr_head_sha, } - merge_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr].merge github_api_put_request(merge_url, github_user, body=body) else: print_warning("Review indicates this PR should not be merged (use -f/--force to do so anyway)") @@ -1197,7 +1251,10 @@ def update_pr(pr, paths, ecs, commit_msg=None): pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') - pr_url = lambda g: g.repos[pr_target_account][pr_target_repo].pulls[pr] + def pr_url(gh): + """Utility function to fetch data for a specific PR.""" + return gh.repos[pr_target_account][pr_target_repo].pulls[pr] + status, pr_data = github_api_get_request(pr_url, github_user) if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", @@ -1324,7 +1381,8 @@ def check_github(): branch_name = 'test_branch_%s' % ''.join(random.choice(string.letters) for _ in range(5)) try: git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=True) - remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'master', silent=True, git_only=True) + remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'master', + silent=True, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) except Exception as err: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4df30e7baa..932a5a8ecd 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -73,7 +73,10 @@ from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, HAVE_GITHUB_API, HAVE_KEYRING +from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN +from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS +from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING from easybuild.tools.github import fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains @@ -131,6 +134,9 @@ def eb_shell_quote(token): DEFAULT_SYS_CFGFILES = [f for d in XDG_CONFIG_DIRS for f in sorted(glob.glob(os.path.join(d, 'easybuild.d', '*.cfg')))] DEFAULT_USER_CFGFILE = os.path.join(XDG_CONFIG_HOME, 'easybuild', 'config.cfg') +DEFAULT_LIST_PR_STATE = GITHUB_PR_STATE_OPEN +DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED +DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC _log = fancylogger.getLogger('options', fname=False) @@ -586,6 +592,9 @@ def github_options(self): 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), 'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False), + 'list-prs': ("List pull requests", str, 'store_or_None', + ",".join([DEFAULT_LIST_PR_STATE, DEFAULT_LIST_PR_ORDER, DEFAULT_LIST_PR_DIREC]), + {'metavar': 'STATE,ORDER,DIRECTION'}), 'merge-pr': ("Merge pull request", int, 'store', None, {'metavar': 'PR#'}), 'new-pr': ("Open a new pull request", None, 'store_true', False), 'pr-branch-name': ("Branch name to use for new PRs; '_new_pr_' if unspecified", @@ -800,6 +809,10 @@ def postprocess(self): if self.options.optarch and not self.options.job: self._postprocess_optarch() + # make sure --list-prs has a valid format + if self.options.list_prs: + self._postprocess_list_prs() + # handle configuration options that affect other configuration options self._postprocess_config() @@ -833,6 +846,27 @@ def _postprocess_optarch(self): else: self.log.info("Keeping optarch raw: %s", self.options.optarch) + def _postprocess_list_prs(self): + """Postprocess --list-prs options""" + list_pr_parts = self.options.list_prs.split(',') + nparts = len(list_pr_parts) + + if nparts > 3: + raise EasyBuildError("Argument to --list-prs must be in the format 'state[,order[,direction]]") + + list_pr_state = list_pr_parts[0] + list_pr_order = list_pr_parts[1] if nparts > 1 else DEFAULT_LIST_PR_ORDER + list_pr_direc = list_pr_parts[2] if nparts > 2 else DEFAULT_LIST_PR_DIREC + + if list_pr_state not in GITHUB_PR_STATES: + raise EasyBuildError("1st item in --list-prs ('%s') must be one of %s", list_pr_state, GITHUB_PR_STATES) + if list_pr_order not in GITHUB_PR_ORDERS: + raise EasyBuildError("2nd item in --list-prs ('%s') must be one of %s", list_pr_order, GITHUB_PR_ORDERS) + if list_pr_direc not in GITHUB_PR_DIRECTIONS: + raise EasyBuildError("3rd item in --list-prs ('%s') must be one of %s", list_pr_direc, GITHUB_PR_DIRECTIONS) + + self.options.list_prs = (list_pr_state, list_pr_order, list_pr_direc) + def _postprocess_include(self): """Postprocess --include options.""" # set up included easyblocks, module naming schemes and toolchains/toolchain components diff --git a/test/framework/github.py b/test/framework/github.py index e9a8f22379..7e97085dfe 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -114,6 +114,22 @@ def test_read(self): except (IOError, OSError): pass + def test_list_prs(self): + """Test list_prs function.""" + if self.github_token is None: + print "Skipping test_list_prs, no GitHub token available?" + return + + parameters = ('closed', 'created', 'asc') + + init_config(build_options={'pr_target_account': GITHUB_USER, + 'pr_target_repo': GITHUB_REPO}) + + expected = "PR #1: a pr" + + output = gh.list_prs(parameters, per_page=1) + self.assertEqual(expected, output) + def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.github_token is None: diff --git a/test/framework/options.py b/test/framework/options.py index 046394ae11..36f4b9f9be 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -40,7 +40,7 @@ import easybuild.tools.build_log import easybuild.tools.options import easybuild.tools.toolchain -from easybuild.framework.easyblock import EasyBlock, FETCH_STEP +from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -50,8 +50,8 @@ from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env from easybuild.tools.filetools import copy_dir, copy_file, download_file, mkdir, read_file, remove_file, write_file -from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, URL_SEPARATOR -from easybuild.tools.github import fetch_github_token +from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod from easybuild.tools.options import EasyBuildOptions, parse_external_modules_metadata, set_tmpdir, use_color from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX @@ -3196,6 +3196,29 @@ def test_use_color(self): easybuild.tools.options.terminal_supports_colors = lambda _: False self.assertFalse(use_color('auto')) + def test_list_prs(self): + """Test --list-prs.""" + args = ['--list-prs', 'foo'] + error_msg = "must be one of \['open', 'closed', 'all'\]" + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + + args = ['--list-prs', 'open,foo'] + error_msg = "must be one of \['created', 'updated', 'popularity', 'long-running'\]" + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + + args = ['--list-prs', 'open,created,foo'] + error_msg = "must be one of \['asc', 'desc'\]" + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + + args = ['--list-prs', 'open,created,asc,foo'] + error_msg = "must be in the format 'state\[,order\[,direction\]\]" + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + + args = ['--list-prs', 'closed,updated,asc'] + txt, _ = self._run_mock_eb(args, testing=False) + expected = "Listing PRs with parameters: direction=asc, per_page=100, sort=updated, state=closed" + self.assertTrue(expected in txt) + def test_list_software(self): """Test --list-software and --list-installed-software.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'v1.0')