diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index ab9e470e0c2c..8b41df6b269f 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -339,9 +339,12 @@ public void showMemoryUsage() { MDC.of(LogKeys.MEMORY_SIZE, memoryNotAccountedFor), MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId)); logger.info( - "{} bytes of memory are used for execution and {} bytes of memory are used for storage", + "{} bytes of memory are used for execution " + + "and {} bytes of memory are used for storage " + + "and {} bytes of unmanaged memory are used", MDC.of(LogKeys.EXECUTION_MEMORY_SIZE, memoryManager.executionMemoryUsed()), - MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed())); + MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed()), + MDC.of(LogKeys.MEMORY_SIZE, UnifiedMemoryManager$.MODULE$.getUnmanagedMemoryUsed())); } } diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala index db51f14415e1..6b278c47f32f 100644 --- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala @@ -273,6 +273,14 @@ object UnifiedMemoryManager extends Logging { // Atomic flag to ensure polling is only started once per JVM private val pollingStarted = new AtomicBoolean(false) + /** + * Returns the total unmanaged memory in bytes, including both + * on-heap unmanaged memory and off-heap unmanaged memory. + */ + private[spark] def getUnmanagedMemoryUsed: Long = { + UnifiedMemoryManager.unmanagedOnHeapUsed.get() + UnifiedMemoryManager.unmanagedOffHeapUsed.get() + } + /** * Register an unmanaged memory consumer to track its memory usage. *