diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py index b5761713be..a290de3f42 100644 --- a/easybuild/scripts/clean_gists.py +++ b/easybuild/scripts/clean_gists.py @@ -32,9 +32,10 @@ from easybuild.base.generaloption import simple_option from easybuild.base.rest import RestClient from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_API_URL, HTTP_STATUS_OK, GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO from easybuild.tools.github import GITHUB_EB_MAIN, fetch_github_token from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.py2vs3 import HTTPError, URLError HTTP_DELETE_OK = 204 @@ -49,6 +50,7 @@ def main(): 'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'), 'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'), 'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'), + 'dry-run': ("Only show which gists will be deleted but don't actually delete them", None, 'store_true', False), } go = simple_option(options) @@ -58,6 +60,7 @@ def main(): raise EasyBuildError("Please tell me what to do?") if go.options.github_user is None: + EasyBuildOptions.DEFAULT_LOGLEVEL = None # Don't overwrite log level eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[]) username = eb_go.options.github_user log.debug("Fetch github username from easybuild, found: %s", username) @@ -88,7 +91,8 @@ def main(): break log.info("Found %s gists", len(all_gists)) - regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P[0-9]+))?\)?$") + re_eb_gist = re.compile(r"(EasyBuild test report|EasyBuild log for failed build)(.*?)$") + re_pr_nr = re.compile(r"(EB )?PR #([0-9]+)") pr_cache = {} num_deleted = 0 @@ -96,43 +100,79 @@ def main(): for gist in all_gists: if not gist["description"]: continue - re_pr_num = regex.search(gist["description"]) - delete_gist = False - - if re_pr_num: - log.debug("Found a Easybuild gist (id=%s)", gist["id"]) - pr_num = re_pr_num.group("PR") - if go.options.all: - delete_gist = True - elif pr_num and go.options.closed_pr: - log.debug("Found Easybuild test report for PR #%s", pr_num) - - if pr_num not in pr_cache: - status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get() + + gist_match = re_eb_gist.search(gist["description"]) + + if not gist_match: + log.debug("Found a non-Easybuild gist (id=%s)", gist["id"]) + continue + + log.debug("Found an Easybuild gist (id=%s)", gist["id"]) + + pr_data = gist_match.group(2) + + pr_nrs_matches = re_pr_nr.findall(pr_data) + + if go.options.all: + delete_gist = True + elif not pr_nrs_matches: + log.debug("Found Easybuild test report without PR (id=%s).", gist["id"]) + delete_gist = go.options.orphans + elif go.options.closed_pr: + # All PRs must be closed + delete_gist = True + for pr_nr_match in pr_nrs_matches: + eb_str, pr_num = pr_nr_match + if eb_str or GITHUB_EASYBLOCKS_REPO in pr_data: + repo = GITHUB_EASYBLOCKS_REPO + else: + repo = GITHUB_EASYCONFIGS_REPO + + cache_key = "%s-%s" % (repo, pr_num) + + if cache_key not in pr_cache: + try: + status, pr = gh.repos[GITHUB_EB_MAIN][repo].pulls[pr_num].get() + except HTTPError as e: + status, pr = e.code, e.msg if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s", pr_num, status, pr) - pr_cache[pr_num] = pr["state"] - - if pr_cache[pr_num] == "closed": - log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"]) - delete_gist = True - - elif not pr_num and go.options.orphans: - log.debug("Found Easybuild test report without PR (id=%s)", gist["id"]) - delete_gist = True + pr_cache[cache_key] = pr["state"] + + if pr_cache[cache_key] == "closed": + log.debug("Found report from closed %s PR #%s (id=%s)", repo, pr_num, gist["id"]) + elif delete_gist: + if len(pr_nrs_matches) > 1: + log.debug("Found at least 1 PR, that is not closed yet: %s/%s (id=%s)", + repo, pr_num, gist["id"]) + delete_gist = False + else: + delete_gist = True if delete_gist: - status, del_gist = gh.gists[gist["id"]].delete() + if go.options.dry_run: + log.info("DRY-RUN: Delete gist with id=%s", gist["id"]) + num_deleted += 1 + continue + try: + status, del_gist = gh.gists[gist["id"]].delete() + except HTTPError as e: + status, del_gist = e.code, e.msg + except URLError as e: + status, del_gist = None, e.reason if status != HTTP_DELETE_OK: - raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s", - gist["id"], status, del_gist) + log.warning("Unable to remove gist (id=%s): error code %s, message = %s", + gist["id"], status, del_gist) else: - log.info("Delete gist with id=%s", gist["id"]) + log.info("Deleted gist with id=%s", gist["id"]) num_deleted += 1 - log.info("Deleted %s gists", num_deleted) + if go.options.dry_run: + log.info("DRY-RUN: Would delete %s gists", num_deleted) + else: + log.info("Deleted %s gists", num_deleted) if __name__ == '__main__': diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index cf93034570..93f62babb1 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -138,22 +138,30 @@ def session_state(): } -def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): +def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False, easyblock_pr_nrs=None): """Create test report for easyconfigs PR, in Markdown format.""" github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO + pr_target_repo = build_option('pr_target_repo') end_time = gmtime() # create a gist with a full test report test_report = [] if pr_nr is not None: + repo = pr_target_repo or GITHUB_EASYCONFIGS_REPO test_report.extend([ - "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, pr_nr), "", ]) + if easyblock_pr_nrs: + repo = pr_target_repo or GITHUB_EASYBLOCKS_REPO + test_report.extend([ + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, repo, nr) + for nr in easyblock_pr_nrs + ]) + test_report.append("") test_report.extend([ "#### Test result", "%s" % msg, @@ -184,6 +192,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l descr = "(partial) EasyBuild log for failed build of %s" % ec['spec'] if pr_nr is not None: descr += " (PR #%s)" % pr_nr + if easyblock_pr_nrs: + descr += "".join(" (easyblock PR #%s)" % nr for nr in easyblock_pr_nrs) fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url @@ -318,20 +328,21 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state """ dump_path = build_option('dump_test_report') pr_nr = build_option('from_pr') - eb_pr_nrs = build_option('include_easyblocks_from_pr') + easyblock_pr_nrs = build_option('include_easyblocks_from_pr') upload = build_option('upload_test_report') if upload: msg = msg + " (%d easyconfigs in total)" % orig_cnt - test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True) + test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nr=pr_nr, gist_log=True, + easyblock_pr_nrs=easyblock_pr_nrs) if pr_nr: # upload test report to gist and issue a comment in the PR to notify txt = post_pr_test_report(pr_nr, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state, success) - elif eb_pr_nrs: + elif easyblock_pr_nrs: # upload test report to gist and issue a comment in the easyblocks PR to notify - for eb_pr_nr in map(int, eb_pr_nrs): - txt = post_pr_test_report(eb_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, init_session_state, - success) + for easyblock_pr_nr in map(int, easyblock_pr_nrs): + txt = post_pr_test_report(easyblock_pr_nr, GITHUB_EASYBLOCKS_REPO, test_report, msg, + init_session_state, success) else: # only upload test report as a gist gist_url = upload_test_report_as_gist(test_report['full'])