Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 58 additions & 43 deletions .github/scripts/auto-backport.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
#!/usr/bin/env python3

import argparse
import logging
import os
import re
import sys
import tempfile
import logging

from git import GitCommandError, Repo
from github import Github, GithubException
from git import Repo, GitCommandError

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
try:
github_token = os.environ["GITHUB_TOKEN"]
except KeyError:
Expand All @@ -19,32 +20,39 @@


def is_pull_request():
return '--pull-request' in sys.argv[1:]
return "--pull-request" in sys.argv[1:]


def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--repo', type=str, required=True, help='Github repository name')
parser.add_argument('--base-branch', type=str, default='refs/heads/next', help='Base branch')
parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged')
parser.add_argument('--label', type=str, required=is_pull_request(), help='Backport label name when --pull-request is defined')
parser.add_argument("--repo", type=str, required=True, help="Github repository name")
parser.add_argument("--base-branch", type=str, default="refs/heads/next", help="Base branch")
parser.add_argument("--commits", default=None, type=str, help="Range of promoted commits.")
parser.add_argument("--pull-request", type=int, help="Pull request number to be backported")
parser.add_argument(
"--head-commit",
type=str,
required=is_pull_request(),
help="The HEAD of target branch after the pull request specified by --pull-request is merged",
)
parser.add_argument(
"--label", type=str, required=is_pull_request(), help="Backport label name when --pull-request is defined"
)
return parser.parse_args()


def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
pr_body = f'{pr.body}\n\n'
pr_body = f"{pr.body}\n\n"
for commit in commits:
pr_body += f'- (cherry picked from commit {commit})\n\n'
pr_body += f'Parent PR: #{pr.number}'
pr_body += f"- (cherry picked from commit {commit})\n\n"
pr_body += f"Parent PR: #{pr.number}"
try:
backport_pr = repo.create_pull(
title=backport_pr_title,
body=pr_body,
head=f'scylladbbot:{new_branch_name}',
head=f"scylladbbot:{new_branch_name}",
base=base_branch_name,
draft=is_draft
draft=is_draft,
)
logging.info(f"Pull request created: {backport_pr.html_url}")
backport_pr.add_to_assignees(pr.user)
Expand All @@ -56,10 +64,10 @@ def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr
logging.info(f"Assigned PR to original author: {pr.user}")
return backport_pr
except GithubException as e:
if 'A pull request already exists' in str(e):
logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}')
if "A pull request already exists" in str(e):
logging.warning(f"A pull request already exists for {pr.user}:{new_branch_name}")
else:
logging.error(f'Failed to create PR: {e}')
logging.error(f"Failed to create PR: {e}")


def get_pr_commits(repo, pr, stable_branch, start_commit=None):
Expand All @@ -84,58 +92,65 @@ def get_pr_commits(repo, pr, stable_branch, start_commit=None):
if promoted_commit.commit.message.startswith(commit_title):
commits.append(promoted_commit.sha)

elif pr.state == 'closed':
elif pr.state == "closed":
events = pr.get_issue_events()
for event in events:
if event.event == 'closed':
if event.event == "closed":
commits.append(event.commit_id)
return commits


def backport(repo, pr, version, commits, backport_base_branch):
new_branch_name = f'backport/{pr.number}/to-{version}'
backport_pr_title = f'[Backport {version}] {pr.title}'
repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
fork_repo = f'https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git'
with (tempfile.TemporaryDirectory() as local_repo_path):
new_branch_name = f"backport/{pr.number}/to-{version}"
backport_pr_title = f"[Backport {version}] {pr.title}"
repo_url = f"https://scylladbbot:{github_token}@github.com/{repo.full_name}.git"
fork_repo = f"https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git"
with tempfile.TemporaryDirectory() as local_repo_path:
try:
repo_local = Repo.clone_from(repo_url, local_repo_path, branch=backport_base_branch)
repo_local.git.checkout(b=new_branch_name)
is_draft = False
for commit in commits:
try:
repo_local.git.cherry_pick(commit, '-m1', '-x')
repo_local.git.cherry_pick(commit, "-m1", "-x")
except GitCommandError as e:
logging.warning(f'Cherry-pick conflict on commit {commit}: {e}')
logging.warning(f"Cherry-pick conflict on commit {commit}: {e}")
is_draft = True
repo_local.git.add(A=True)
repo_local.git.cherry_pick('--continue')
repo_local.git.cherry_pick("--continue")
repo_local.git.push(fork_repo, new_branch_name, force=True)
create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
is_draft=is_draft)
create_pull_request(
repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits, is_draft=is_draft
)
except GitCommandError as e:
logging.warning(f"GitCommandError: {e}")


def main():
args = parse_args()
base_branch = args.base_branch.split('/')[2]
promoted_label = 'promoted-to-master'
base_branch = args.base_branch.split("/")[2]
promoted_label = "promoted-to-master"
repo_name = args.repo
if 'scylla-enterprise-machine-image' in args.repo:
promoted_label = 'promoted-to-enterprise'

backport_branch = 'next-'
stable_branch = 'master' if base_branch == 'next' else 'enterprise' if base_branch == 'next-enterprise' else base_branch.replace('next', 'branch')
backport_label_pattern = re.compile(r'backport/\d+\.\d+$')
if "scylla-enterprise-machine-image" in args.repo:
promoted_label = "promoted-to-enterprise"

backport_branch = "next-"
stable_branch = (
"master"
if base_branch == "next"
else "enterprise"
if base_branch == "next-enterprise"
else base_branch.replace("next", "branch")
)
backport_label_pattern = re.compile(r"backport/\d+\.\d+$")

g = Github(github_token)
repo = g.get_repo(repo_name)
closed_prs = []
start_commit = None

if args.commits:
start_commit, end_commit = args.commits.split('..')
start_commit, end_commit = args.commits.split("..")
commits = repo.compare(start_commit, end_commit).commits
for commit in commits:
for pr in commit.get_pulls():
Expand All @@ -152,16 +167,16 @@ def main():
else:
backport_labels = [label for label in labels if backport_label_pattern.match(label)]
if promoted_label not in labels:
print(f'no {promoted_label} label: {pr.number}')
print(f"no {promoted_label} label: {pr.number}")
continue
if not backport_labels:
print(f'no backport label: {pr.number}')
print(f"no backport label: {pr.number}")
continue
commits = get_pr_commits(repo, pr, stable_branch, start_commit)
logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
for backport_label in backport_labels:
version = backport_label.replace('backport/', '')
backport_base_branch = backport_label.replace('backport/', backport_branch)
version = backport_label.replace("backport/", "")
backport_base_branch = backport_label.replace("backport/", backport_branch)
backport(repo, pr, version, commits, backport_base_branch)


Expand Down
51 changes: 24 additions & 27 deletions .github/scripts/search_commits.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#!/usr/bin/env python3

import argparse
import os
import re
import sys

import requests
from github import Github
import argparse
import sys
import os


try:
github_token = os.environ["GITHUB_TOKEN"]
Expand All @@ -16,57 +18,52 @@

def parser():
parser = argparse.ArgumentParser()
parser.add_argument('--repository', type=str, default='scylladb/scylla-pkg', help='Github repository name')
parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.')
parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use')
parser.add_argument('--ref', type=str, required=True, help='PR target branch')
parser.add_argument("--repository", type=str, default="scylladb/scylla-pkg", help="Github repository name")
parser.add_argument("--commits", type=str, required=True, help="Range of promoted commits.")
parser.add_argument("--label", type=str, default="promoted-to-master", help="Label to use")
parser.add_argument("--ref", type=str, required=True, help="PR target branch")
return parser.parse_args()


def main():
args = parser()
g = Github(github_token)
repo = g.get_repo(args.repository, lazy=False)
start_commit, end_commit = args.commits.split('..')
start_commit, end_commit = args.commits.split("..")
commits = repo.compare(start_commit, end_commit).commits
processed_prs = set()
for commit in commits:
search_url = f'https://api.github.com/search/issues'
search_url = "https://api.github.com/search/issues"
query = f"repo:{args.repository} is:pr is:merged sha:{commit.sha}"
params = {
"q": query,
}
headers = {
"Authorization": f"token {github_token}",
"Accept": "application/vnd.github.v3+json"
}
headers = {"Authorization": f"token {github_token}", "Accept": "application/vnd.github.v3+json"}
response = requests.get(search_url, headers=headers, params=params)
prs = response.json().get("items", [])
for pr in prs:
match = re.findall(r'Parent PR: #(\d+)', pr["body"])
match = re.findall(r"Parent PR: #(\d+)", pr["body"])
if match:
pr_number = int(match[0])
if pr_number in processed_prs:
continue
ref = re.search(r'-(\d+\.\d+)', args.ref)
label_to_add = f'backport/{ref.group(1)}-done'
label_to_remove = f'backport/{ref.group(1)}'
remove_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels/{label_to_remove}'
del_data = {
"labels": [f'{label_to_remove}']
}
ref = re.search(r"-(\d+\.\d+)", args.ref)
label_to_add = f"backport/{ref.group(1)}-done"
label_to_remove = f"backport/{ref.group(1)}"
remove_label_url = (
f"https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels/{label_to_remove}"
)
del_data = {"labels": [f"{label_to_remove}"]}
response = requests.delete(remove_label_url, headers=headers, json=del_data)
if response.ok:
print(f'Label {label_to_remove} removed successfully')
print(f"Label {label_to_remove} removed successfully")
else:
print(f'Label {label_to_remove} cant be removed')
print(f"Label {label_to_remove} cant be removed")
else:
pr_number = pr["number"]
label_to_add = args.label
data = {
"labels": [f'{label_to_add}']
}
add_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels'
data = {"labels": [f"{label_to_add}"]}
add_label_url = f"https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels"
response = requests.post(add_label_url, headers=headers, json=data)
if response.ok:
print(f"Label added successfully to {add_label_url}")
Expand Down
13 changes: 10 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,18 @@ jobs:
python-version: 3.14
architecture: x64

- name: unittest
- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Run pre-commit
run: |
uv sync
uv run pre-commit run --all-files

- name: unittest
run: |
pip install -r test-requirements.txt
pytest ./tests
uv sync
uv run pytest ./tests

- name: Build RPM (Rockylinux:8)
run: docker run -v `pwd`:/scylla-machine-image -w /scylla-machine-image --rm rockylinux:8 bash -c 'dnf update -y; dnf install -y git ; git config --global --add safe.directory "*"; ./dist/redhat/build_rpm.sh -t centos8'
Expand Down
12 changes: 12 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.8.4
hooks:
# Run the linter.
- id: ruff
args: [--fix]
# Run the formatter.
- id: ruff-format
1 change: 1 addition & 0 deletions .python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.14
Loading