From 3c63869195f28a39830e099188146210e6110630 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Fri, 3 Feb 2023 14:07:45 +0800
Subject: [PATCH 01/11] [ci] Kill hanged docker build process to avoid build
 timeout issue.

---
 .azure-pipelines/azure-pipelines-build.yml | 20 ++++++++++++++++++++
 .azure-pipelines/template-variables.yml    |  1 +
 2 files changed, 21 insertions(+)

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index 09a46573fe0..9bc86baee45 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -110,6 +110,25 @@ jobs:
     buildSteps:
       - template: template-skipvstest.yml
       - bash: |
+          (
+            while true
+            do
+              sleep 120
+              now=$(date +%s)
+              pids=$(ps aux | grep -v grep | grep -E "^.{,100}docker build" | awk '{print$2}')
+              for pid in $pids
+              do
+                start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+                time_s=$(($now-$start))
+                if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+                  echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+                  ps $pid &>> target/daemon.log
+                  sudo kill $pid
+                fi
+              done
+            done
+          ) &
+          daemon_pid=$!
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
@@ -167,5 +186,6 @@ jobs:
             fi
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
+          sudo kill $daemon_pid
         displayName: "Build sonic image"
       - template: check-dirty-version.yml
diff --git a/.azure-pipelines/template-variables.yml b/.azure-pipelines/template-variables.yml
index ce2ec687431..42d46182d26 100644
--- a/.azure-pipelines/template-variables.yml
+++ b/.azure-pipelines/template-variables.yml
@@ -5,3 +5,4 @@ variables:
   SONIC_BUILD_RETRY_COUNT: 3
   SONIC_BUILD_RETRY_INTERVAL: 600
   DOCKER_BUILDKIT: 0
+  DOCKER_BUILD_TIMEOUT: 3600

From cb271312d4f3c34cb95f17abf9cd8f41085854a4 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Mon, 13 Feb 2023 15:16:08 +0800
Subject: [PATCH 02/11] fix

---
 .azure-pipelines/azure-pipelines-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index 9bc86baee45..d4e42fa74f8 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -115,7 +115,7 @@ jobs:
             do
               sleep 120
               now=$(date +%s)
-              pids=$(ps aux | grep -v grep | grep -E "^.{,100}docker build" | awk '{print$2}')
+              pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f1)
               for pid in $pids
               do
                 start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)

From f81681a18a49593805797df4518af341e4f28953 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Wed, 15 Feb 2023 21:48:17 +0800
Subject: [PATCH 03/11] use template

---
 .azure-pipelines/azure-pipelines-build.yml | 22 ++-------------------
 .azure-pipelines/template-daemon.yml       | 23 ++++++++++++++++++++++
 .azure-pipelines/template-stop-daemon.yml  |  4 ++++
 3 files changed, 29 insertions(+), 20 deletions(-)
 create mode 100644 .azure-pipelines/template-daemon.yml
 create mode 100644 .azure-pipelines/template-stop-daemon.yml

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index d4e42fa74f8..ee582fc2417 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -109,26 +109,8 @@ jobs:
 
     buildSteps:
       - template: template-skipvstest.yml
+      - template: .azure-pipelines/template-daemon.yml@buildimage
       - bash: |
-          (
-            while true
-            do
-              sleep 120
-              now=$(date +%s)
-              pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f1)
-              for pid in $pids
-              do
-                start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
-                time_s=$(($now-$start))
-                if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
-                  echo =========== $(date +%F%T) $time_s &>> target/daemon.log
-                  ps $pid &>> target/daemon.log
-                  sudo kill $pid
-                fi
-              done
-            done
-          ) &
-          daemon_pid=$!
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
@@ -186,6 +168,6 @@ jobs:
             fi
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
-          sudo kill $daemon_pid
         displayName: "Build sonic image"
+      - template: .azure-pipelines/template-stop-daemon.yml@buildimage
       - template: check-dirty-version.yml
diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
new file mode 100644
index 00000000000..fe8553f4b41
--- /dev/null
+++ b/.azure-pipelines/template-daemon.yml
@@ -0,0 +1,23 @@
+steps:
+- script: |
+    (
+      while true
+      do
+        sleep 120
+        now=$(date +%s)
+        pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f1)
+        for pid in $pids
+        do
+          start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+          time_s=$(($now-$start))
+          if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+            echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+            ps $pid &>> target/daemon.log
+            sudo kill $pid
+          fi
+        done
+      done
+    ) &
+    daemon_pid=$!
+    ps $daemon_pid
+    echo "##vso[task.setvariable variable=DAEMON_PID]$daemon_pid"
diff --git a/.azure-pipelines/template-stop-daemon.yml b/.azure-pipelines/template-stop-daemon.yml
new file mode 100644
index 00000000000..1fdd2a54ed5
--- /dev/null
+++ b/.azure-pipelines/template-stop-daemon.yml
@@ -0,0 +1,4 @@
+scripts:
+- script: |
+    ps $(DAEMON_PID)
+    sudo kill $(DAEMON_PID) || true

From c60c51d56858df3c9b956a5f6c85b637d4ea0fc7 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Wed, 15 Feb 2023 21:51:41 +0800
Subject: [PATCH 04/11] fix

---
 .azure-pipelines/template-daemon.yml      | 2 +-
 .azure-pipelines/template-stop-daemon.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
index fe8553f4b41..7c02604e3e4 100644
--- a/.azure-pipelines/template-daemon.yml
+++ b/.azure-pipelines/template-daemon.yml
@@ -1,5 +1,5 @@
 steps:
-- script: |
+- bash: |
     (
       while true
       do
diff --git a/.azure-pipelines/template-stop-daemon.yml b/.azure-pipelines/template-stop-daemon.yml
index 1fdd2a54ed5..a5987fec4f2 100644
--- a/.azure-pipelines/template-stop-daemon.yml
+++ b/.azure-pipelines/template-stop-daemon.yml
@@ -1,4 +1,4 @@
-scripts:
-- script: |
+steps:
+- bash: |
     ps $(DAEMON_PID)
     sudo kill $(DAEMON_PID) || true

From ab51e63efc8584d564bcaa3caff13798d03303bc Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Wed, 15 Feb 2023 21:59:16 +0800
Subject: [PATCH 05/11] fix

---
 .azure-pipelines/azure-pipelines-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index ee582fc2417..79f811ea8e7 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -109,7 +109,7 @@ jobs:
 
     buildSteps:
       - template: template-skipvstest.yml
-      - template: .azure-pipelines/template-daemon.yml@buildimage
+      - template: template-daemon.yml
       - bash: |
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
@@ -169,5 +169,5 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
-      - template: .azure-pipelines/template-stop-daemon.yml@buildimage
+      - template: template-stop-daemon.yml
       - template: check-dirty-version.yml

From 241a43561a439b448b2e6fec36f3e756fe836b9b Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 11:17:01 +0800
Subject: [PATCH 06/11] fix

---
 .azure-pipelines/template-daemon.yml      | 1 +
 .azure-pipelines/template-stop-daemon.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
index 7c02604e3e4..a49fc268a49 100644
--- a/.azure-pipelines/template-daemon.yml
+++ b/.azure-pipelines/template-daemon.yml
@@ -21,3 +21,4 @@ steps:
     daemon_pid=$!
     ps $daemon_pid
     echo "##vso[task.setvariable variable=DAEMON_PID]$daemon_pid"
+  displayName: start daemon to kill hang docker
diff --git a/.azure-pipelines/template-stop-daemon.yml b/.azure-pipelines/template-stop-daemon.yml
index a5987fec4f2..cc3cacebfbc 100644
--- a/.azure-pipelines/template-stop-daemon.yml
+++ b/.azure-pipelines/template-stop-daemon.yml
@@ -2,3 +2,4 @@ steps:
 - bash: |
     ps $(DAEMON_PID)
     sudo kill $(DAEMON_PID) || true
+  displayName: kill daemon process

From 85d554ef751d4b661734d4accafdde4c8a0b6353 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 14:07:57 +0800
Subject: [PATCH 07/11] fix

---
 .azure-pipelines/azure-pipelines-build.yml | 1 -
 .azure-pipelines/cleanup.yml               | 4 ++++
 .azure-pipelines/template-stop-daemon.yml  | 5 -----
 3 files changed, 4 insertions(+), 6 deletions(-)
 delete mode 100644 .azure-pipelines/template-stop-daemon.yml

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index 79f811ea8e7..ba3227a349d 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -169,5 +169,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
-      - template: template-stop-daemon.yml
       - template: check-dirty-version.yml
diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml
index 6569c8ed0bf..88245990c7b 100644
--- a/.azure-pipelines/cleanup.yml
+++ b/.azure-pipelines/cleanup.yml
@@ -1,5 +1,9 @@
 steps:
 - script: |
+    # kill daemon process
+    ps $(PIPELINE_WATCHDOG_DAEMON_PID)
+    sudo kill $(PIPELINE_WATCHDOG_DAEMON_PID)
+
     if sudo [ -f /var/run/march/docker.pid ] ; then
         pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
     fi
diff --git a/.azure-pipelines/template-stop-daemon.yml b/.azure-pipelines/template-stop-daemon.yml
deleted file mode 100644
index cc3cacebfbc..00000000000
--- a/.azure-pipelines/template-stop-daemon.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-steps:
-- bash: |
-    ps $(DAEMON_PID)
-    sudo kill $(DAEMON_PID) || true
-  displayName: kill daemon process

From ae4c756b8ec07e34534bf137ab283bfdee925a1c Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 14:08:42 +0800
Subject: [PATCH 08/11] fix

---
 .azure-pipelines/template-daemon.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
index a49fc268a49..9527e1dc091 100644
--- a/.azure-pipelines/template-daemon.yml
+++ b/.azure-pipelines/template-daemon.yml
@@ -20,5 +20,5 @@ steps:
     ) &
     daemon_pid=$!
     ps $daemon_pid
-    echo "##vso[task.setvariable variable=DAEMON_PID]$daemon_pid"
+    echo "##vso[task.setvariable variable=PIPELINE_WATCHDOG_DAEMON_PID]$daemon_pid"
   displayName: start daemon to kill hang docker

From 71565d12a79d29ce76cd19f38e9e6f49bb027cb6 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 14:14:19 +0800
Subject: [PATCH 09/11] fix

---
 .azure-pipelines/cleanup.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml
index 88245990c7b..940624fa280 100644
--- a/.azure-pipelines/cleanup.yml
+++ b/.azure-pipelines/cleanup.yml
@@ -15,4 +15,5 @@ steps:
         pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
     fi
     sudo rm -rf $(ls -A1)
+  condition: always()
   displayName: "Clean Workspace"

From ea5ea5898d5a7d3c57fdd8246e79e16b363c18fb Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 14:16:35 +0800
Subject: [PATCH 10/11] fix

---
 .azure-pipelines/cleanup.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml
index 940624fa280..3cfac914164 100644
--- a/.azure-pipelines/cleanup.yml
+++ b/.azure-pipelines/cleanup.yml
@@ -1,5 +1,6 @@
 steps:
 - script: |
+    set -x
     # kill daemon process
     ps $(PIPELINE_WATCHDOG_DAEMON_PID)
     sudo kill $(PIPELINE_WATCHDOG_DAEMON_PID)

From 3988ed5fb4dfc3eb12f72e378195ecf20476f723 Mon Sep 17 00:00:00 2001
From: shilongliu <shilongliu@microsoft.com>
Date: Thu, 16 Feb 2023 16:24:29 +0800
Subject: [PATCH 11/11] fix

---
 .azure-pipelines/cleanup.yml         | 5 +++--
 .azure-pipelines/template-daemon.yml | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.azure-pipelines/cleanup.yml b/.azure-pipelines/cleanup.yml
index 3cfac914164..4f74de0f413 100644
--- a/.azure-pipelines/cleanup.yml
+++ b/.azure-pipelines/cleanup.yml
@@ -2,8 +2,9 @@ steps:
 - script: |
     set -x
     # kill daemon process
-    ps $(PIPELINE_WATCHDOG_DAEMON_PID)
-    sudo kill $(PIPELINE_WATCHDOG_DAEMON_PID)
+    ps $(cat /tmp/azp_daemon_kill_docker_pid)
+    sudo kill $(cat /tmp/azp_daemon_kill_docker_pid)
+    rm /tmp/azp_daemon_kill_docker_pid
 
     if sudo [ -f /var/run/march/docker.pid ] ; then
         pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
index 9527e1dc091..b0915557ced 100644
--- a/.azure-pipelines/template-daemon.yml
+++ b/.azure-pipelines/template-daemon.yml
@@ -20,5 +20,5 @@ steps:
     ) &
     daemon_pid=$!
     ps $daemon_pid
-    echo "##vso[task.setvariable variable=PIPELINE_WATCHDOG_DAEMON_PID]$daemon_pid"
+    echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid
   displayName: start daemon to kill hang docker