diff --git a/build_debian.sh b/build_debian.sh
index e3e19c61cbe..ca17268e16f 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -470,6 +470,9 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
+
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index 3022546a068..6a638f58c00 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 {% if docker_config_engine_buster_debs.strip() %}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_config_engine_buster_debs.split(' '), "/debs/") }}
diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2
index 21536f43c01..da3fd11208c 100755
--- a/dockers/docker-platform-monitor/Dockerfile.j2
+++ b/dockers/docker-platform-monitor/Dockerfile.j2
@@ -53,6 +53,11 @@ RUN pip3 install grpcio==1.39.0 \
 RUN pip2 install thrift==0.13.0
 RUN pip3 install thrift==0.13.0
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+# The the PyYAML for python3 has already installed in the base image docker-config-engine
+RUN pip2 install wheel==0.37.1
+RUN pip2 install PyYAML==5.4.1 --no-build-isolation
+
 {% if docker_platform_monitor_debs.strip() -%}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_platform_monitor_debs.split(' '), "/debs/") }}
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index c6e7cf19a0a..7bb558c2e04 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -418,7 +418,8 @@ RUN pip3 uninstall -y enum34
 RUN pip3 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 RUN pip3 install "lxml==4.6.2"
 
 # For sonic-platform-common testing
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index fc119cb8d1d..0de9696a206 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -446,8 +446,9 @@ RUN pip3 install MarkupSafe==2.0.1
 RUN pip3 install Jinja2==3.0.3
 
 # For sonic-mgmt-framework
-RUN pip2 install "PyYAML==5.4.1"
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 RUN pip2 install "lxml==4.6.5"
 RUN pip3 install "lxml==4.6.5"