From c356b1a28899279b68d64a3292f829d12a044140 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 11 Jun 2025 14:59:10 -0700 Subject: [PATCH] [SPARK-52456][CORE] Lower the maximum size limit of event log files --- .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++-- .../spark/deploy/history/EventLogFileWritersSuite.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 7cb3d068b676..000de4d6c26d 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -317,8 +317,8 @@ package object config { " to be rolled over.") .version("3.0.0") .bytesConf(ByteUnit.BYTE) - .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " + - "configured to be at least 10 MiB.") + .checkValue(_ >= ByteUnit.MiB.toBytes(2), "Max file size of event log should be " + + "configured to be at least 2 MiB.") .createWithDefaultString("128m") private[spark] val EXECUTOR_ID = diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala index 7c05613d8b16..da3614bf81a5 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala @@ -326,7 +326,7 @@ class RollingEventLogFilesWriterSuite extends EventLogFileWritersSuite { val conf = getLoggingConf(testDirPath, None) conf.set(EVENT_LOG_ENABLE_ROLLING, true) - conf.set(EVENT_LOG_ROLLING_MAX_FILE_SIZE.key, "9m") + conf.set(EVENT_LOG_ROLLING_MAX_FILE_SIZE.key, "1m") val e = intercept[IllegalArgumentException] { createWriter(appId, attemptId, testDirPath.toUri, conf,