diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index f0cf710981b2a..b35c0bcea4218 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -640,6 +640,10 @@ private void revertFromOld(Path oldPath, boolean fileExisted)
   private void renameOrFail(Path src, Path dest)
       throws IOException {
     if (!fs.rename(src, dest)) {
+      if (!fs.exists(src)) {
+        throw new FileNotFoundException(src.toUri().toString());
+      }
+
       throw new IOException("Rename unsuccessful : "
           + String.format("'%s' to '%s'", src, dest));
     }