obtain PR/commit diff via GitHub API rather than downloading *.diff file via github.com

easybuild/tools/github.py

@@ -54,7 +54,7 @@
 from easybuild.framework.easyconfig.parser import EasyConfigParser
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg, print_warning
-from easybuild.tools.config import DEFAULT_DOWNLOAD_MAX_ATTEMPTS, build_option
+from easybuild.tools.config import build_option
 from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files
 from easybuild.tools.filetools import det_patched_files, download_file, extract_file
 from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file
@@ -548,15 +548,29 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
     diff_url = pr_data['diff_url']
     diff_fn = os.path.basename(diff_url)
     diff_filepath = os.path.join(path, diff_fn)
-    # max. 6 attempts + initial wait time of 10sec -> max. 10 * (2^6) = 640sec (~10min) before giving up on download
-    # see also https://github.com/easybuilders/easybuild-framework/issues/4869
-    max_attempts = DEFAULT_DOWNLOAD_MAX_ATTEMPTS
-    download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False,
-                  max_attempts=max_attempts)
-    if not os.path.exists(diff_filepath):
-        raise EasyBuildError(f"Failed to download {diff_url}, even after {max_attempts} attempts and being patient...")
-    diff_txt = read_file(diff_filepath)
-    _log.debug("Diff for PR #%s:\n%s", pr, diff_txt)
+
+    def pr_request_fn(gh):
+        return gh.repos[github_account][github_repo].pulls[pr]
+
+    # see also https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request,
+    # in particular part about media types
+    error_msg = None
+    accept_diff = {'Accept': 'application/vnd.github.diff'}
+    try:
+        status, data = github_api_get_request(pr_request_fn, github_user=github_user, headers=accept_diff)
+        if status == HTTP_STATUS_OK:
+            # decode from bytes to text
+            diff_txt = data.decode()
+            _log.debug("Diff for PR #%s:\n%s", pr, diff_txt)
+            write_file(diff_filepath, diff_txt)
+        else:
+            error_msg = f"HTTP status code: {status}"
+    except HTTPError as err:
+        error_msg = str(err)
+
+    if error_msg:
+        error_msg = f"Failed to download diff for {github_account}/{github_repo} PR #{pr}! ({error_msg})"
+        raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_GITHUB)
 
     patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
     _log.debug("List of patched files for PR #%s: %s", pr, patched_files)
@@ -703,24 +717,32 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
 
     # if no files are specified, determine which files are touched in commit
     if not files:
-        diff_url = os.path.join(GITHUB_URL, github_account, github_repo, 'commit', commit + '.diff')
-        diff_fn = os.path.basename(diff_url)
-        diff_filepath = os.path.join(path, diff_fn)
-        # max. 6 attempts + initial wait time of 10sec -> max. 10 * (2^6) = 640sec (~10min) before giving up on download
-        # see also https://github.com/easybuilders/easybuild-framework/issues/4869
-        max_attempts = DEFAULT_DOWNLOAD_MAX_ATTEMPTS
-        download_file(diff_fn, diff_url, diff_filepath, forced=True, trace=False,
-                      max_attempts=max_attempts)
-        if os.path.exists(diff_filepath):
-            diff_txt = read_file(diff_filepath)
-            _log.debug("Diff for commit %s:\n%s", commit, diff_txt)
-
-            files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
-            _log.debug("List of patched files for commit %s: %s", commit, files)
-        else:
-            msg = f"Failed to download diff for commit {commit} of {github_account}/{github_repo} "
-            msg += " (after {max_attempts} attempts)"
-            raise EasyBuildError(msg, exit_code=EasyBuildExit.FAIL_GITHUB)
+        github_user = build_option('github_user')
+
+        def commit_request_fn(gh):
+            return gh.repos[github_account][github_repo].commits[commit]
+
+        error_msg = None
+        try:
+            # see also https://docs.github.com/en/rest/commits/commits#get-a-commit,
+            # in particular part about media types
+            accept_diff = {'Accept': 'application/vnd.github.diff'}
+            status, data = github_api_get_request(commit_request_fn, github_user=github_user, headers=accept_diff)
+            if status == HTTP_STATUS_OK:
+                # decode from bytes to text
+                diff_txt = data.decode()
+                _log.debug("Diff for commit %s:\n%s", commit, diff_txt)
+            else:
+                error_msg = f"HTTP status code: {status}"
+        except HTTPError as err:
+            error_msg = str(err)
+
+        if error_msg:
+            error_msg = f"Failed to download diff for {github_account}/{github_repo} commit {commit}! ({error_msg})"
+            raise EasyBuildError(error_msg, exit_code=EasyBuildExit.FAIL_GITHUB)
+
+        files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True, filter_deleted=True)
+        _log.debug("List of patched files for commit %s: %s", commit, files)
 
     # download tarball for specific commit
     repo_commit = download_repo(repo=github_repo, commit=commit, account=github_account)

test/framework/github.py

@@ -548,7 +548,7 @@ def test_fetch_files_from_commit(self):
         self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, '7c83a55')
 
         # test downloading of non-existing commit
-        error_pattern = r"Failed to download diff for commit c0ff33c0ff33 of easybuilders/easybuild-easyconfigs"
+        error_pattern = r"Failed to download diff for easybuilders/easybuild-easyconfigs commit c0ff33c0ff33"
         self.assertErrorRegex(EasyBuildError, error_pattern, fetch_files_from_commit, 'c0ff33c0ff33')
 
     def test_fetch_easyconfigs_from_commit(self):