From 06be5e9cc888064f72c49bdb467497f3d6654cc2 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 1 May 2016 19:53:59 -0700 Subject: [PATCH 1/2] Update TaskMemoryManager.java Issue number SPARK-15045. Removed Dead-code. --- .../java/org/apache/spark/memory/TaskMemoryManager.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index 2796114fc545a..a3fa8c19d0c1d 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -389,12 +389,6 @@ public long cleanUpAllAllocatedMemory() { consumers.clear(); } - for (MemoryBlock page : pageTable) { - if (page != null) { - memoryManager.tungstenMemoryAllocator().free(page); - } - } - Arrays.fill(pageTable, null); // release the memory that is not used by any consumer. memoryManager.releaseExecutionMemory(acquiredButNotUsed, taskAttemptId, tungstenMemoryMode); From ad748dd95174f4f7df126a1d0a5a1273540ec9a4 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Wed, 4 May 2016 20:26:08 -0700 Subject: [PATCH 2/2] Updated the new patch by rewriting the synchronized block --- .../java/org/apache/spark/memory/TaskMemoryManager.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index a3fa8c19d0c1d..89521c1d6c307 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -379,7 +379,6 @@ public long getOffsetInPage(long pagePlusOffsetAddress) { */ public long cleanUpAllAllocatedMemory() { synchronized (this) { - Arrays.fill(pageTable, null); for (MemoryConsumer c: consumers) { if (c != null && c.getUsed() > 0) { // In case of failed task, it's normal to see leaked memory @@ -387,6 +386,14 @@ public long cleanUpAllAllocatedMemory() { } } consumers.clear(); + + for (MemoryBlock page : pageTable) { + if (page != null) { + logger.warn("leak a page: " + page + " in task " + taskAttemptId); + memoryManager.tungstenMemoryAllocator().free(page); + } + } + Arrays.fill(pageTable, null); }