Skip to content

Commit d50f332

Browse files
xushiyanxushiyan
committed
[HUDI-4005] Update release scripts to help validation
1 parent 33ff475 commit d50f332

3 files changed

Lines changed: 77 additions & 4 deletions

File tree

‎scripts/release/deploy_staging_jars.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ declare -a ALL_VERSION_OPTS=(
4343
"-Dscala-2.11 -Dspark2.4 -Dflink1.13"
4444
"-Dscala-2.11 -Dspark2.4 -Dflink1.14"
4545
"-Dscala-2.12 -Dspark2.4 -Dflink1.13"
46-
"-Dscala-2.12 -Dspark3.1 -Dflink1.14"
4746
"-Dscala-2.12 -Dspark3.2 -Dflink1.14"
47+
"-Dscala-2.12 -Dspark3.1 -Dflink1.14" # run this last to make sure utilities bundle has spark 3.1
4848
)
4949
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"
5050

‎scripts/release/validate_staged_bundles.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
22+
# fail immediately
23+
set -o errexit
24+
set -o nounset
25+
26+
REPO=$1
27+
VERSION=$2
28+
29+
STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi"
30+
31+
declare -a BUNDLE_URLS=(
32+
"${STAGING_REPO}/hudi-datahub-sync-bundle/${VERSION}/hudi-datahub-sync-bundle-${VERSION}.jar"
33+
"${STAGING_REPO}/hudi-flink1.13-bundle_2.11/${VERSION}/hudi-flink1.13-bundle_2.11-${VERSION}.jar"
34+
"${STAGING_REPO}/hudi-flink1.13-bundle_2.12/${VERSION}/hudi-flink1.13-bundle_2.12-${VERSION}.jar"
35+
"${STAGING_REPO}/hudi-flink1.14-bundle_2.11/${VERSION}/hudi-flink1.14-bundle_2.11-${VERSION}.jar"
36+
"${STAGING_REPO}/hudi-flink1.14-bundle_2.12/${VERSION}/hudi-flink1.14-bundle_2.12-${VERSION}.jar"
37+
"${STAGING_REPO}/hudi-gcp-bundle/${VERSION}/hudi-gcp-bundle-${VERSION}.jar"
38+
"${STAGING_REPO}/hudi-hadoop-mr-bundle/${VERSION}/hudi-hadoop-mr-bundle-${VERSION}.jar"
39+
"${STAGING_REPO}/hudi-hive-sync-bundle/${VERSION}/hudi-hive-sync-bundle-${VERSION}.jar"
40+
"${STAGING_REPO}/hudi-integ-test-bundle/${VERSION}/hudi-integ-test-bundle-${VERSION}.jar"
41+
"${STAGING_REPO}/hudi-kafka-connect-bundle/${VERSION}/hudi-kafka-connect-bundle-${VERSION}.jar"
42+
"${STAGING_REPO}/hudi-presto-bundle/${VERSION}/hudi-presto-bundle-${VERSION}.jar"
43+
"${STAGING_REPO}/hudi-spark-bundle_2.11/${VERSION}/hudi-spark-bundle_2.11-${VERSION}.jar"
44+
"${STAGING_REPO}/hudi-spark-bundle_2.12/${VERSION}/hudi-spark-bundle_2.12-${VERSION}.jar"
45+
"${STAGING_REPO}/hudi-spark2.4-bundle_2.11/${VERSION}/hudi-spark2.4-bundle_2.11-${VERSION}.jar"
46+
"${STAGING_REPO}/hudi-spark2.4-bundle_2.12/${VERSION}/hudi-spark2.4-bundle_2.12-${VERSION}.jar"
47+
"${STAGING_REPO}/hudi-spark3-bundle_2.12/${VERSION}/hudi-spark3-bundle_2.12-${VERSION}.jar"
48+
"${STAGING_REPO}/hudi-spark3.1-bundle_2.12/${VERSION}/hudi-spark3.1-bundle_2.12-${VERSION}.jar"
49+
"${STAGING_REPO}/hudi-spark3.2-bundle_2.12/${VERSION}/hudi-spark3.2-bundle_2.12-${VERSION}.jar"
50+
"${STAGING_REPO}/hudi-timeline-server-bundle/${VERSION}/hudi-timeline-server-bundle-${VERSION}.jar"
51+
"${STAGING_REPO}/hudi-trino-bundle/${VERSION}/hudi-trino-bundle-${VERSION}.jar"
52+
"${STAGING_REPO}/hudi-utilities-bundle_2.11/${VERSION}/hudi-utilities-bundle_2.11-${VERSION}.jar"
53+
"${STAGING_REPO}/hudi-utilities-bundle_2.12/${VERSION}/hudi-utilities-bundle_2.12-${VERSION}.jar"
54+
"${STAGING_REPO}/hudi-utilities-slim-bundle_2.11/${VERSION}/hudi-utilities-slim-bundle_2.11-${VERSION}.jar"
55+
"${STAGING_REPO}/hudi-utilities-slim-bundle_2.12/${VERSION}/hudi-utilities-slim-bundle_2.12-${VERSION}.jar"
56+
)
57+
58+
NOW=$(date +%s)
59+
TMP_DIR_FOR_BUNDLES=/tmp/${NOW}
60+
mkdir "$TMP_DIR_FOR_BUNDLES"
61+
for url in "${BUNDLE_URLS[@]}"; do
62+
echo "downloading $url"
63+
wget "$url" -P "$TMP_DIR_FOR_BUNDLES"
64+
done
65+
66+
ls -l "$TMP_DIR_FOR_BUNDLES"

‎scripts/release/validate_staged_release.sh‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,8 @@ pushd $WORK_DIR
7878

7979
# Checkout dist repo
8080
LOCAL_SVN_DIR=local_svn_dir
81-
ROOT_SVN_URL=https://dist.apache.org/repos/dist/
81+
ROOT_SVN_URL=https://dist.apache.org/repos/dist
8282
REPO_TYPE=${RELEASE_TYPE}
83-
#RELEASE_REPO=release
8483
HUDI_REPO=hudi
8584

8685
if [ $RC_NUM == -1 ]; then
@@ -99,7 +98,15 @@ echo "Downloading from svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO}"
9998
(bash -c "svn co ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} $REDIRECT") || (echo -e "\t\t Unable to checkout ${ROOT_SVN_URL}/${REPO_TYPE}/${HUDI_REPO} to $REDIRECT. Please run with --verbose to get details\n" && exit -1)
10099

101100
echo "Validating hudi-${ARTIFACT_SUFFIX} with release type \"${REPO_TYPE}\""
102-
cd ${HUDI_REPO}/hudi-${ARTIFACT_SUFFIX}
101+
if [ $RELEASE_TYPE == "release" ]; then
102+
ARTIFACT_PREFIX=
103+
elif [ $RELEASE_TYPE == "dev" ]; then
104+
ARTIFACT_PREFIX='hudi-'
105+
else
106+
echo "Unexpected RELEASE_TYPE: $RELEASE_TYPE"
107+
exit 1;
108+
fi
109+
cd ${HUDI_REPO}/${ARTIFACT_PREFIX}${ARTIFACT_SUFFIX}
103110
$SHASUM hudi-${ARTIFACT_SUFFIX}.src.tgz > got.sha512
104111

105112
echo "Checking Checksum of Source Release"

0 commit comments

Comments
 (0)