Skip to content

Commit a348664

Browse files
EmmonsCursesunlei1024
authored andcommitted
[CI] Optimize ce_build for RL_E2E tasks (PaddlePaddle#7670)
1 parent fee7c0b commit a348664

2 files changed

Lines changed: 64 additions & 3 deletions

File tree

.github/workflows/_build_linux_rl.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,12 @@ jobs:
165165
chown -R $(whoami) /workspace/FastDeploy
166166
cd FastDeploy
167167
168-
python -m pip uninstall paddlepaddle-gpu -y || true
169-
wget -q --no-proxy https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
170-
python -m pip install paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
168+
# When `PADDLE_WHL_URL` is not empty, download PaddlePaddle installation package and install it, otherwise use the default PaddlePaddle.
169+
if [[ -n "${PADDLE_WHL_URL}" ]]; then
170+
python -m pip uninstall paddlepaddle-gpu -y || true
171+
wget -q --no-proxy ${PADDLE_WHL_URL}
172+
python -m pip install paddlepaddle_gpu*.whl
173+
fi
171174
172175
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
173176

.github/workflows/ce_job.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,18 @@ jobs:
188188
FD_VERSION: 0.0.0
189189
PADDLE_WHL_URL: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
190190

191+
build_sm90100_rl:
192+
name: BUILD_SM90100_RL
193+
needs: [clone, ce_job_pre_check]
194+
if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
195+
uses: ./.github/workflows/_build_linux_rl.yml
196+
with:
197+
DOCKER_IMAGE: iregistry.baidu-int.com/paddlepaddle/base-images:paddlecloud-ubuntu24.04-gcc13.3-cuda12.9-cudnn9.9-bccl1.4.1.4-nccl2.26.5-openmpi4.1.5-FleetY13.4.1H-rc1
198+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
199+
COMPILE_ARCH: "90,100"
200+
WITH_NIGHTLY_BUILD: OFF
201+
FD_VERSION: 0.0.0
202+
191203
build_sm8689:
192204
name: BUILD_SM8689
193205
needs: [clone, ce_job_pre_check]
@@ -299,6 +311,52 @@ jobs:
299311
echo "commit wheel url is ${WHEEL_PATH}"
300312
echo "latest wheel url is ${WHEEL_PATH_LATEST}"
301313
314+
ce_upload_sm90100_rl:
315+
environment: CodeSync
316+
name: CE_UPLOAD_90100RL
317+
needs: build_sm90100_rl
318+
runs-on: ubuntu-latest
319+
env:
320+
AK: ${{ secrets.BOS_AK }}
321+
SK: ${{ secrets.BOS_SK }}
322+
FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm90100_rl.outputs.wheel_path_rl }}
323+
COMPILE_ARCH: "90,100"
324+
steps:
325+
- uses: actions/setup-python@v6
326+
with:
327+
python-version: '3.10'
328+
- name: Wheel Info Show and Upload
329+
run: |
330+
echo "The wheel is located at: ${{ needs.build_sm90100_rl.outputs.wheel_path_rl }}"
331+
wget -q --no-check-certificate ${{ needs.build_sm90100_rl.outputs.wheel_path_rl }}
332+
filename=$(basename ${{ needs.build_sm90100_rl.outputs.wheel_path_rl }})
333+
334+
commit_id=${{ github.sha }}
335+
branch_name=${{ github.ref_name }}
336+
337+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
338+
push_file=$(realpath bos_tools.py)
339+
python -m pip install bce-python-sdk==0.9.29
340+
341+
target_paths=(
342+
"paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_90100/${branch_name}/${commit_id}"
343+
"paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_90100/${branch_name}/latest"
344+
)
345+
346+
for target_path in "${target_paths[@]}"; do
347+
echo "Uploading ${filename} to ${target_path}"
348+
python "${push_file}" "${filename}" "${target_path}"
349+
done
350+
351+
base_prefix="paddle-qa/"
352+
commit_path_stripped="${target_paths[0]#${base_prefix}}"
353+
latest_path_stripped="${target_paths[1]#${base_prefix}}"
354+
WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}"
355+
WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}"
356+
357+
echo "commit wheel url is ${WHEEL_PATH}"
358+
echo "latest wheel url is ${WHEEL_PATH_LATEST}"
359+
302360
ce_upload_sm8689:
303361
environment: CodeSync
304362
name: CE_UPLOAD

0 commit comments

Comments
 (0)