[SPARK-53057][CORE] Support sizeOf in SparkFileUtils and JavaUtils

### What changes were proposed in this pull request?

This PR aims to support `sizeOf` in `SparkFileUtils` and `JavaUtils`.

### Why are the changes needed?

To provide a better implementation than `FileUtils.sizeOf` and `FileUtils.sizeOfDirectory`.

```scala
scala> spark.time(org.apache.spark.network.util.JavaUtils.sizeOf(new java.io.File("/tmp/spark")))
Time taken: 143 ms
val res0: Long = 732733895

scala> spark.time(org.apache.commons.io.FileUtils.sizeOf(new java.io.File("/tmp/spark")))
Time taken: 208 ms
val res1: Long = 732733895
```

### Does this PR introduce _any_ user-facing change?

No behavior change.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #51765 from dongjoon-hyun/SPARK-53057.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>

Author: dongjoon-hyun

common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java

@@ -23,9 +23,13 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.FileVisitResult;
+import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -82,6 +86,25 @@ public static String bytesToString(ByteBuffer b) {
     return StandardCharsets.UTF_8.decode(b.slice()).toString();
   }
 
+  public static long sizeOf(File file) throws IOException {
+    if (!file.exists()) {
+      throw new IllegalArgumentException(file.getAbsolutePath() + " not found");
+    }
+    return sizeOf(file.toPath());
+  }
+
+  public static long sizeOf(Path dirPath) throws IOException {
+    AtomicLong size = new AtomicLong(0);
+    Files.walkFileTree(dirPath, new SimpleFileVisitor<Path>() {
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+          size.addAndGet(attrs.size());
+          return FileVisitResult.CONTINUE;
+        }
+      });
+    return size.get();
+  }
+
   /**
    * Delete a file or directory and its contents recursively.
    * Don't follow directories if they are symlinks.

common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala

@@ -49,6 +49,13 @@ private[spark] trait SparkFileUtils extends Logging {
     new File(path).getCanonicalFile().toURI()
   }
 
+  /**
+   * Size of files recursively.
+   */
+  def sizeOf(f: File): Long = {
+    JavaUtils.sizeOf(f)
+  }
+
   /**
    * Lists files recursively.
    */

connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala

@@ -760,12 +760,12 @@ abstract class AvroSuite
       spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, ZSTANDARD.lowerCaseName())
       df.write.format("avro").save(zstandardDir)
 
-      val uncompressSize = FileUtils.sizeOfDirectory(new File(uncompressDir))
-      val bzip2Size = FileUtils.sizeOfDirectory(new File(bzip2Dir))
-      val xzSize = FileUtils.sizeOfDirectory(new File(xzDir))
-      val deflateSize = FileUtils.sizeOfDirectory(new File(deflateDir))
-      val snappySize = FileUtils.sizeOfDirectory(new File(snappyDir))
-      val zstandardSize = FileUtils.sizeOfDirectory(new File(zstandardDir))
+      val uncompressSize = Utils.sizeOf(new File(uncompressDir))
+      val bzip2Size = Utils.sizeOf(new File(bzip2Dir))
+      val xzSize = Utils.sizeOf(new File(xzDir))
+      val deflateSize = Utils.sizeOf(new File(deflateDir))
+      val snappySize = Utils.sizeOf(new File(snappyDir))
+      val zstandardSize = Utils.sizeOf(new File(zstandardDir))
 
       assert(uncompressSize > deflateSize)
       assert(snappySize > deflateSize)

core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala

@@ -286,7 +286,7 @@ private class HistoryServerDiskManager(
   }
 
   /** Visible for testing. Return the size of a directory. */
-  private[history] def sizeOf(path: File): Long = FileUtils.sizeOf(path)
+  private[history] def sizeOf(path: File): Long = Utils.sizeOf(path)
 
   private[history] class Lease(val tmpPath: File, private val leased: Long) {
 

scalastyle-config.xml

@@ -292,6 +292,11 @@ This file is divided into 3 sections:
     <customMessage>Use java.nio.file.Files.readAllBytes</customMessage>
   </check>
 
+  <check customId="sizeOf" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
+    <parameters><parameter name="regex">FileUtils\.sizeOf(Directory)?</parameter></parameters>
+    <customMessage>Use sizeOf of JavaUtils or Utils instead.</customMessage>
+  </check>
+
   <check customId="commonslang2" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
     <parameters><parameter name="regex">org\.apache\.commons\.lang\.</parameter></parameters>
     <customMessage>Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead