From 94e091928f1ccfb9cee3ba9b6d9d37e0a56a4fed Mon Sep 17 00:00:00 2001
From: XieYunshen <1084314248@qq.com>
Date: Mon, 31 Aug 2020 13:13:48 +0800
Subject: [PATCH 1/3] add retry on mac

---
 paddle/scripts/paddle_build.sh | 53 ++++++++++++++++++++++++++++++++--
 1 file changed, 51 insertions(+), 2 deletions(-)

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 926747ef6186e3..4ba0358dde2617 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -528,8 +528,44 @@ EOF
         elif [ "$1" == "cp37-cp37m" ]; then
             pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
         fi
+        tmp_dir=`mktemp -d`
+        tmpfile_rand=`date +%s%N`
+        tmpfile=$tmp_dir/$tmpfile_rand
         ut_startTime_s=`date +%s`
-        ctest --output-on-failure -j $2;mactest_error=$?
+        ctest --output-on-failure -j $2 | tee $tmpfile
+        failed_test_lists=''
+        collect_failed_tests
+        #rm -f $tmp_dir/*
+        retry_unittests_record=''
+        retry_time=3
+        exec_times=0
+        exec_time_array=('first' 'second' 'third')
+        if [ -n "$failed_test_lists" ];then
+            mactest_error=1
+            while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
+                do
+                    retry_unittests_record="$retry_unittests_record$failed_test_lists"
+                    failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
+                    read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(\w+\)" | sed 's/(.\+)//' | sed 's/- //' )
+                    echo "========================================="
+                    echo "This is the ${exec_time_array[$exec_times]} time to re-run"
+                    echo "========================================="
+                    echo "The following unittest will be re-run:"
+                    echo "${failed_test_lists_ult}"
+
+                    for line in ${retry_unittests[@]} ;
+                        do
+                            if [[ "$retry_unittests_regular" = "" ]];then
+                                retry_unittests_regular="^$line$"
+                            else
+                                retry_unittests_regular="$retry_unittests_regular|^$line$"
+                        done
+                    rm -f $tmp_dir/*
+                    failed_test_lists=''
+                    ctest -R "($retry_unittests_regular)" --output-on-failure -j $2 | tee $tmpfile
+                    collect_failed_tests
+                    retry_unittests_regular=''
+        #mactest_error=$?
         ut_endTime_s=`date +%s`
         echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
         paddle version
@@ -537,7 +573,20 @@ EOF
         export http_proxy=$my_proxy
         export https_proxy=$my_proxy
         if [ "$mactest_error" != 0 ];then
-            exit 8;
+            if [[ "$failed_test_lists" == "" ]]; then
+                echo "========================================"
+                echo "There are failed tests, which have been successful after re-run:"
+                echo "========================================"
+                echo "The following tests have been re-ran:"
+                echo "${retry_unittests_record}"
+            else
+                failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
+                echo "========================================"
+                echo "Summary Failed Tests... "
+                echo "========================================"
+                echo "The following tests FAILED: "
+                echo "${failed_test_lists_ult}"
+                exit 8;
         fi
     fi
 }

From e637a5f067ba208f60115d25f648c1e22c14eda9 Mon Sep 17 00:00:00 2001
From: XieYunshen <1084314248@qq.com>
Date: Mon, 31 Aug 2020 14:12:04 +0800
Subject: [PATCH 2/3] fix some error

---
 paddle/scripts/paddle_build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 4ba0358dde2617..d3eb6371afcb1f 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -559,13 +559,15 @@ EOF
                                 retry_unittests_regular="^$line$"
                             else
                                 retry_unittests_regular="$retry_unittests_regular|^$line$"
+                            fi
                         done
                     rm -f $tmp_dir/*
                     failed_test_lists=''
                     ctest -R "($retry_unittests_regular)" --output-on-failure -j $2 | tee $tmpfile
                     collect_failed_tests
                     retry_unittests_regular=''
-        #mactest_error=$?
+                done
+        fi
         ut_endTime_s=`date +%s`
         echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
         paddle version
@@ -587,6 +589,7 @@ EOF
                 echo "The following tests FAILED: "
                 echo "${failed_test_lists_ult}"
                 exit 8;
+            fi
         fi
     fi
 }

From 7dd8234fce79e82425822623ddbc0ff70ca39313 Mon Sep 17 00:00:00 2001
From: XieYunshen <1084314248@qq.com>
Date: Thu, 3 Sep 2020 11:35:04 +0800
Subject: [PATCH 3/3] fix with some errors

---
 paddle/scripts/paddle_build.sh | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index d3eb6371afcb1f..efc63f94946241 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -528,14 +528,14 @@ EOF
         elif [ "$1" == "cp37-cp37m" ]; then
             pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
         fi
-        tmp_dir=`mktemp -d`
         tmpfile_rand=`date +%s%N`
         tmpfile=$tmp_dir/$tmpfile_rand
         ut_startTime_s=`date +%s`
         ctest --output-on-failure -j $2 | tee $tmpfile
         failed_test_lists=''
         collect_failed_tests
-        #rm -f $tmp_dir/*
+        set +x
+        mactest_error=0
         retry_unittests_record=''
         retry_time=3
         exec_times=0
@@ -545,17 +545,19 @@ EOF
             while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
                 do
                     retry_unittests_record="$retry_unittests_record$failed_test_lists"
-                    failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
-                    read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(\w+\)" | sed 's/(.\+)//' | sed 's/- //' )
+                    failed_test_lists_ult=`echo "${failed_test_lists}"`
+                    read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
                     echo "========================================="
                     echo "This is the ${exec_time_array[$exec_times]} time to re-run"
                     echo "========================================="
                     echo "The following unittest will be re-run:"
-                    echo "${failed_test_lists_ult}"
+                    echo "${retry_unittests}"
+                    echo "========================================="
 
+                    retry_unittests_regular=''
                     for line in ${retry_unittests[@]} ;
                         do
-                            if [[ "$retry_unittests_regular" = "" ]];then
+                            if [[ "$retry_unittests_regular" == "" ]];then
                                 retry_unittests_regular="^$line$"
                             else
                                 retry_unittests_regular="$retry_unittests_regular|^$line$"
@@ -565,9 +567,10 @@ EOF
                     failed_test_lists=''
                     ctest -R "($retry_unittests_regular)" --output-on-failure -j $2 | tee $tmpfile
                     collect_failed_tests
-                    retry_unittests_regular=''
+                    exec_times=$[$exec_times+1]
                 done
         fi
+        #mactest_error=$?
         ut_endTime_s=`date +%s`
         echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
         paddle version
@@ -582,7 +585,7 @@ EOF
                 echo "The following tests have been re-ran:"
                 echo "${retry_unittests_record}"
             else
-                failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
+                failed_test_lists_ult=`echo "${failed_test_lists}"`
                 echo "========================================"
                 echo "Summary Failed Tests... "
                 echo "========================================"