diff --git a/paddle/phi/backends/xpu/xpu_context.cc b/paddle/phi/backends/xpu/xpu_context.cc
index e3931d3da19b19..ad0047b4e9ad63 100644
--- a/paddle/phi/backends/xpu/xpu_context.cc
+++ b/paddle/phi/backends/xpu/xpu_context.cc
@@ -151,7 +151,15 @@ struct XPUContext::Impl {
     LOG_FIRST_N(WARNING, 1)
         << "Please NOTE: xpu device: " << static_cast<int>(place_.device);
     context_ = xpu::create_context();
-    context_->set_option("XPUAPI_DEFAULT_SIZE", "1");
+    // Setup XPU GM Buffer
+    if (std::getenv("XPUAPI_DEFAULT_SIZE") != nullptr) {
+      context_->set_option("XPUAPI_DEFAULT_SIZE",
+                           std::getenv("XPUAPI_DEFAULT_SIZE"));
+    } else {
+      // Optimization described in
+      // https://github.com/PaddlePaddle/Paddle/pull/54674
+      context_->set_option("XPUAPI_DEFAULT_SIZE", "1");
+    }
     xpu_version_ = backends::xpu::get_xpu_version(place_.device);
     SetL3Cache();
   }