diff --git a/build_debian.sh b/build_debian.sh
index 5f8c0286b26..b652cdb2983 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -531,9 +531,6 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
-
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-bullseye/Dockerfile.j2 b/dockers/docker-config-engine-bullseye/Dockerfile.j2
index 700af660a91..f6804de903b 100644
--- a/dockers/docker-config-engine-bullseye/Dockerfile.j2
+++ b/dockers/docker-config-engine-bullseye/Dockerfile.j2
@@ -23,10 +23,6 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
-# Fix armhf build failure
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install PyYAML==5.4.1 --no-build-isolation
-
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index cfa61bc0ba8..084956eed5f 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,10 +23,6 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
-# Fix armhf build failure
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install PyYAML==5.4.1 --no-build-isolation
-
 # Install python-redis
 RUN pip3 install redis==4.5.4
 
diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2
index d9517ca7d5e..b6f97fa1700 100644
--- a/sonic-slave-bullseye/Dockerfile.j2
+++ b/sonic-slave-bullseye/Dockerfile.j2
@@ -590,8 +590,7 @@ RUN pip3 uninstall -y enum34
 RUN pip3 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
+
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip3 install "lxml==4.9.1"
 {%- endif %}
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index b65a1a08957..2aa36d1b9c4 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -551,9 +551,6 @@ RUN pip3 install MarkupSafe==2.0.1
 RUN pip3 install Jinja2==3.0.3
 
 # For sonic-mgmt-framework
-# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
-RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
-RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 
 {%- if CROSS_BUILD_ENVIRON != "y" %}
 RUN pip2 install "lxml==4.9.1"