Skip to content

refactor: clean up automq_release.py for PEP 8 compliance and maintainability #3239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SujayPro wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refactor: clean up automq_release.py for PEP 8 compliance and maintainability #3239

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 61 additions & 60 deletions automq_release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# !/usr/bin/python3
#!/usr/bin/python3

"""Release automation script for AutoMQ Kafka.

Handles version tagging, branch creation, and release artifact preparation
for both Kafka and S3Stream components.
"""

import argparse
import logging
import re
import subprocess
import sys
import tempfile
import time
from pathlib import Path
import argparse
from shutil import which
from typing import Optional

logger = logging.getLogger(__name__)

main_branch = "develop"


def fail(msg):
print(msg)
def fail(msg: str) -> None:
logger.error(msg)
sys.exit(1)


def print_output(output):
if output is None or len(output) == 0:
def print_output(output: Optional[str]) -> None:
if not output:
return
for line in output.split('\n'):
print(">", line)
logger.info("> %s", line)


def cmd(action, cmd_arg, *args, **kwargs):
def cmd(action: str, cmd_arg: "str | list[str]", *args, **kwargs) -> None:
if isinstance(cmd_arg, str) and not kwargs.get("shell", False):
cmd_arg = cmd_arg.split()
allow_failure = kwargs.pop("allow_failure", False)
Expand All @@ -52,7 +62,7 @@ def cmd(action, cmd_arg, *args, **kwargs):
stdin.seek(0)
kwargs["stdin"] = stdin

print(action, cmd_arg, stdin_log)
logger.info("%s %s %s", action, cmd_arg, stdin_log)
try:
output = subprocess.check_output(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs)
print_output(output.decode('utf-8'))
Expand All @@ -62,26 +72,27 @@ def cmd(action, cmd_arg, *args, **kwargs):
if num_retries > 0:
kwargs['num_retries'] = num_retries - 1
kwargs['allow_failure'] = allow_failure
print("Retrying... %d remaining retries" % (num_retries - 1))
time.sleep(4. / (num_retries + 1)) # e.g., if retries=3, sleep for 1s, 1.3s, 2s
logger.warning("Retrying... %d remaining retries", num_retries - 1)
time.sleep(4. / (num_retries + 1))
return cmd(action, cmd_arg, *args, **kwargs)

if allow_failure:
return

print("*************************************************")
print("*** First command failure occurred here. ***")
print("*************************************************")
logger.error("*************************************************")
logger.error("*** First command failure occurred here. ***")
logger.error("*************************************************")
fail("")


def cmd_output(cmd, *args, **kwargs):
if isinstance(cmd, str):
cmd = cmd.split()
return subprocess.check_output(cmd, *args, stderr=subprocess.STDOUT, **kwargs).decode('utf-8')
def run_cmd(command: "str | list[str]", *args, **kwargs) -> str:
"""Run a command and return its decoded stdout output."""
if isinstance(command, str):
command = command.split()
return subprocess.check_output(command, *args, stderr=subprocess.STDOUT, **kwargs).decode('utf-8')


def replace(path, pattern, replacement):
def replace(path: str, pattern: str, replacement: str) -> None:
updated = []
with open(path, 'r') as f:
for line in f:
Expand All @@ -92,7 +103,7 @@ def replace(path, pattern, replacement):
f.write(line)


def regexReplace(path, pattern, replacement):
def regex_replace(path: str, pattern: str, replacement: str) -> None:
updated = []
with open(path, 'r') as f:
for line in f:
Expand All @@ -103,97 +114,87 @@ def regexReplace(path, pattern, replacement):
f.write(line)


def is_tool(name):
def is_tool(name: str) -> bool:
"""Check whether `name` is on PATH and marked as executable."""

# from whichcraft import which
from shutil import which

return which(name) is not None


def check_tools(tools):
"""Check whether tools are available in the system"""
def check_tools(tools: "list[str]") -> None:
"""Check whether required tools are available in the system."""
for tool in tools:
if not is_tool(tool):
raise Exception("%s is not installed" % tool)
raise Exception(f"{tool} is not installed")


def is_valid_kafka_tag(tag):
"""Check whether the tag is a valid kafka tag"""
def is_valid_kafka_tag(tag: str) -> bool:
"""Check whether the tag is a valid Kafka tag (e.g. '1.2.3' or '1.2.3-rc1')."""
pattern = r'^\d+\.\d+\.\d+(-rc\d+)?$'
return re.match(pattern, tag) is not None


def is_valid_s3stream_tag(tag):
"""Check whether the tag is a valid s3stream tag"""
def is_valid_s3stream_tag(tag: str) -> bool:
"""Check whether the tag is a valid S3Stream tag (e.g. '1.2.3-s3stream' or '1.2.3-s3stream-rc1')."""
pattern = r'^\d+\.\d+\.\d+-s3stream(-rc\d+)?$'
return re.match(pattern, tag) is not None


def get_project_path():
# Path object for the current file
current_file = Path(__file__)
# Get the directory of the current file
return current_file.parent


def do_release(tag, s3stream_tag):
def do_release(tag: str, s3stream_tag: str) -> str:
new_branch = f"release-{tag}-{int(time.time())}"
cmd(f"Checking out to release branch", f'git checkout --quiet -b {new_branch}')
cmd("Checking out to release branch", f'git checkout --quiet -b {new_branch}')

print("Updating docker compose")
regexReplace("docker/docker-compose.yaml", "image: automqinc/automq:.*$", f"image: automqinc/automq:{tag}")
logger.info("Updating docker compose")
regex_replace("docker/docker-compose.yaml", "image: automqinc/automq:.*$", f"image: automqinc/automq:{tag}")
replace("gradle/dependencies.gradle", ' branch = "main"', f' require "{s3stream_tag}"')

cmd("Committing changes", ["git", "commit", "--all", "--gpg-sign", "--signoff", "--message", f"ci: bump version to {tag}"])
cmd("Pushing changes", ["git", "push", "--quiet", "origin", new_branch])

cmd("Tagging %s" % tag, ["git", "tag", "--sign", "--annotate", tag, "--message", f"release {tag}"])
cmd("Pushing tag %s" % tag, ["git", "push", "--quiet", "origin", tag])
cmd(f"Tagging {tag}", ["git", "tag", "--sign", "--annotate", tag, "--message", f"release {tag}"])
cmd(f"Pushing tag {tag}", ["git", "push", "--quiet", "origin", tag])

return new_branch


def tag_exists(tag):
maybe_tag = cmd_output(f'git tag --list {tag}').strip()
def tag_exists(tag: str) -> bool:
maybe_tag = run_cmd(f'git tag --list {tag}').strip()
return len(maybe_tag) > 0


def check_before_started(tag, s3stream_tag):
def check_before_started(tag: str, s3stream_tag: str) -> None:
check_tools(["git"])
cmd("Fetching latest code", 'git fetch origin')
cmd("Checking workspace clean", 'git diff-index --quiet HEAD --')
starting_branch = cmd_output('git rev-parse --abbrev-ref HEAD').strip()
starting_branch = run_cmd('git rev-parse --abbrev-ref HEAD').strip()
if starting_branch != main_branch:
fail(f"You must run this script on the {main_branch} branch. current: {starting_branch}")
cmd(f"Checking branch {main_branch} up-to-date", f'git diff origin/{main_branch}...{main_branch} --quiet')
cmd(f"Checking tag {s3stream_tag} exists", f'git ls-remote --quiet --exit-code --tags git@github.com:AutoMQ/automq-for-rocketmq.git {s3stream_tag}')
cmd(f"Checking tag {tag} not exists in remote", f'! git ls-remote --quiet --exit-code --tags origin {tag}', shell=True)
print(f"Checking tag {tag} not exists in local")
logger.info("Checking tag %s not exists in local", tag)
if tag_exists(tag):
fail(f"Tag {tag} already exists. Please delete it before running this script.")


if __name__ == '__main__':
parser = argparse.ArgumentParser(description='release AutoMQ Kafka')
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

parser = argparse.ArgumentParser(description='Release AutoMQ Kafka')
parser.add_argument("--kafka-tag", required=True, type=str,
help="release tag, which should be in the format of 'x.y.z' or 'x.y.z-rcN', e.g., 1.2.3 or 11.22.33-rc44")
help="release tag in the format 'x.y.z' or 'x.y.z-rcN', e.g. 1.2.3 or 11.22.33-rc44")
parser.add_argument("--s3stream-tag", required=True, type=str,
help="related s3stream tag, which should have been pushed to the AutoMQ/automq-for-rocketmq repository,\
and should be in the format of 'x.y.z-s3stream' or 'x.y.z-s3stream-rcN', e.g., 1.2.3-s3stream or 11.22.33-s3stream-rc44")
help="related s3stream tag that should already exist in AutoMQ/automq-for-rocketmq, "
"in the format 'x.y.z-s3stream' or 'x.y.z-s3stream-rcN', e.g. 1.2.3-s3stream")

# get tag from command line
args = parser.parse_args()
tag = args.kafka_tag
s3stream_tag = args.s3stream_tag
print(f"=== try to release {tag} ===")
logger.info("=== try to release %s ===", tag)
if not is_valid_kafka_tag(tag):
fail(f"Invalid tag {tag}")
if not is_valid_s3stream_tag(s3stream_tag):
fail(f"Invalid s3stream tag {s3stream_tag}")
check_before_started(tag, s3stream_tag)
branch = do_release(tag, s3stream_tag)
print(f"=== release {tag} done ===")
print(f"Please create a PR to merge release branch to {main_branch} visiting:")
print(f" https://github.com/AutoMQ/automq-for-kafka/pull/new/{main_branch}...{branch}")
logger.info("=== release %s done ===", tag)
logger.info("Please create a PR to merge release branch to %s visiting:", main_branch)
logger.info(" https://github.com/AutoMQ/automq-for-kafka/pull/new/%s...%s", main_branch, branch)