diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index fddd0a4dff..d6f23a6f3b 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -165,36 +165,47 @@ jobs: - name: Checkout Repo GenAIComps uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Check the Validity of Hyperlinks # ignore_links=("https://platform.openai.com/docs/api-reference/fine-tuning" - # "https://platform.openai.com/docs/api-reference/" - # "https://openai.com/index/whisper/" - # "https://platform.openai.com/docs/api-reference/chat/create") + # "https://platform.openai.com/docs/api-reference/" + # "https://openai.com/index/whisper/" + # "https://platform.openai.com/docs/api-reference/chat/create") run: | cd ${{github.workspace}} fail="FALSE" - url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) - if [ -n "$url_lines" ]; then - for url_line in $url_lines; do - url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') - path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) - if [[ "https://platform.openai.com/docs/api-reference/fine-tuning" == "$url" || "https://platform.openai.com/docs/api-reference/" == "$url" || "https://openai.com/index/whisper/" == "$url" || "https://platform.openai.com/docs/api-reference/chat/create" == "$url" ]]; then - echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person." - else - response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response" -ne 200 ]; then - echo "**********Validation failed, try again**********" - response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response_retry" -eq 200 ]; then - echo "*****Retry successfully*****" + merged_commit=$(git log -1 --format='%H') + changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')" + if [ -n "changed_files" ]; then + for changed_file in $changed_files; do + echo $changed_file + url_lines=$(grep -H -Eo '\]\(http[s]?://[^)]+\)' "$changed_file" | grep -Ev 'GenAIComps/blob/main') + if [ -n "$url_lines" ]; then + for url_line in $url_lines; do + url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + if [[ "https://platform.openai.com/docs/api-reference/fine-tuning" == "$url" || "https://platform.openai.com/docs/api-reference/" == "$url" || "https://openai.com/index/whisper/" == "$url" || "https://platform.openai.com/docs/api-reference/chat/create" == "$url" ]]; then + echo "Link "$url" from ${{github.workspace}}/$path need to be verified by a real person." else - echo "Invalid link from ${{github.workspace}}/$path: $url" - fail="TRUE" + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" + fi + fi fi - fi + done fi done + else + echo "No changed .md file." fi if [[ "$fail" == "TRUE" ]]; then @@ -212,6 +223,8 @@ jobs: - name: Checkout Repo GenAIComps uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Checking Relative Path Validity run: | @@ -225,33 +238,34 @@ jobs: branch="https://github.com/opea-project/GenAIComps/blob/${{ github.event.pull_request.head.ref }}" fi link_head="https://github.com/opea-project/GenAIComps/blob/main" + + merged_commit=$(git log -1 --format='%H') + changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')" png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http') if [ -n "$png_lines" ]; then for png_line in $png_lines; do refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) + if [[ "${png_path:0:1}" == "/" ]]; then - check_path=${{github.workspace}}$png_path - elif [[ "${png_path:0:1}" == "#" ]]; then - check_path=${{github.workspace}}/$refer_path$png_path + check_path=$png_path + elif [[ "$png_path" == *#* ]]; then + relative_path=$(echo "$png_path" | cut -d '#' -f1) + if [ -n "$relative_path" ]; then + check_path=$(dirname "$refer_path")/$relative_path + png_path=$(echo "$png_path" | awk -F'#' '{print "#" $2}') + else + check_path=$refer_path + fi else - check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path + check_path=$(dirname "$refer_path")/$png_path fi - real_path=$(realpath $check_path) - if [ $? -ne 0 ]; then - echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" - fail="TRUE" - else - url=$link_head$(echo "$real_path" | sed 's|.*/GenAIComps||') - response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response" -ne 200 ]; then - echo "**********Validation failed, try again**********" - response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") - if [ "$response_retry" -eq 200 ]; then - echo "*****Retry successfully*****" - else - echo "Retry failed. Check branch ${{ github.event.pull_request.head.ref }}" - url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||') + + if [ -e "$check_path" ]; then + real_path=$(realpath $check_path) + if [[ "$png_line" == *#* ]]; then + if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then + url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIComps||')$png_path response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev") if [ "$response" -ne 200 ]; then echo "**********Validation failed, try again**********" @@ -259,14 +273,17 @@ jobs: if [ "$response_retry" -eq 200 ]; then echo "*****Retry successfully*****" else - echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path" + echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path, link: $url_dev" fail="TRUE" fi else - echo "Check branch ${{ github.event.pull_request.head.ref }} successfully." + echo "Validation succeed $png_line" fi fi fi + else + echo "$check_path does not exist" + fail="TRUE" fi done fi