diff --git a/build_debian.sh b/build_debian.sh
index 7325ace2b7..6c63c4423e 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -549,9 +549,6 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
-
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-bullseye/Dockerfile.j2 b/dockers/docker-config-engine-bullseye/Dockerfile.j2
index 4d0345c4b5..f300ca2d05 100644
--- a/dockers/docker-config-engine-bullseye/Dockerfile.j2
+++ b/dockers/docker-config-engine-bullseye/Dockerfile.j2
@@ -23,10 +23,6 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
-# Fix armhf build failure
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install PyYAML==5.4.1 --no-build-isolation
-
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index cfa61bc0ba..084956eed5 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,10 +23,6 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
-# Fix armhf build failure
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install PyYAML==5.4.1 --no-build-isolation
-
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index 17a5a1b99a..6c4e310537 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -581,8 +581,7 @@ RUN pip3 uninstall -y enum34
 RUN pip3 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
+
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip3 install "lxml==4.9.1"
 {%- endif %}
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index dce6daa520..a6fe68ee17 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -549,9 +549,6 @@ RUN pip3 install MarkupSafe==2.0.1
 RUN pip3 install Jinja2==3.0.3
 
 # For sonic-mgmt-framework
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
-RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip2 install "lxml==4.9.1"