Add File.OpenNullHandle() for efficient process I/O redirection (#123483)

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: adamsitnik <[email protected]>
Co-authored-by: stephentoub <[email protected]>
Co-authored-by: Adam Sitnik <[email protected]>

Author: 3 people

src/libraries/System.Private.CoreLib/src/System/IO/File.Unix.cs

@@ -7,6 +7,8 @@ namespace System.IO
 {
     public static partial class File
     {
+        private const string NullDevicePath = "/dev/null";
+
         private static UnixFileMode GetUnixFileModeCore(string path)
             => FileSystem.GetUnixFileMode(Path.GetFullPath(path));
 

src/libraries/System.Private.CoreLib/src/System/IO/File.Windows.cs

@@ -7,6 +7,8 @@ namespace System.IO
 {
     public static partial class File
     {
+        private const string NullDevicePath = "NUL";
+
         private static UnixFileMode GetUnixFileModeCore(string path)
             => throw new PlatformNotSupportedException(SR.PlatformNotSupported_UnixFileMode);
 

src/libraries/System.Private.CoreLib/src/System/IO/File.cs

@@ -170,6 +170,32 @@ public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Op
             return SafeFileHandle.Open(Path.GetFullPath(path), mode, access, share, options, preallocationSize);
         }
 
+        /// <summary>
+        /// Opens a handle to the system's null device.
+        /// </summary>
+        /// <returns>A <see cref="SafeFileHandle"/> to the system's null device.</returns>
+        /// <remarks>
+        /// <para>
+        /// On Windows, this opens a handle to "NUL". On Unix-based systems, this opens a handle to "/dev/null".
+        /// </para>
+        /// <para>
+        /// The null device is a special file that discards all data written to it and provides no data (EOF)
+        /// when read from. This is useful for redirecting unwanted output or providing empty input to processes.
+        /// </para>
+        /// <para>
+        /// The returned handle supports both reading and writing. All read operations return 0 (EOF), and all
+        /// write operations succeed without storing any data.
+        /// </para>
+        /// <para>
+        /// For scenarios that don't require raw file handles or descriptors, consider using <see cref="Stream.Null"/> instead.
+        /// </para>
+        /// </remarks>
+        /// <exception cref="IOException">An I/O error occurred while opening the null device.</exception>
+        public static SafeFileHandle OpenNullHandle()
+        {
+            return SafeFileHandle.Open(NullDevicePath, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, FileOptions.None, preallocationSize: 0);
+        }
+
         // File and Directory UTC APIs treat a DateTimeKind.Unspecified as UTC whereas
         // ToUniversalTime treats this as local.
         internal static DateTimeOffset GetUtcDateTimeOffset(DateTime dateTime)

src/libraries/System.Runtime/ref/System.Runtime.cs

@@ -10359,6 +10359,7 @@ public static void Move(string sourceFileName, string destFileName, bool overwri
         public static System.IO.FileStream Open(string path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share) { throw null; }
         public static System.IO.FileStream Open(string path, System.IO.FileStreamOptions options) { throw null; }
         public static Microsoft.Win32.SafeHandles.SafeFileHandle OpenHandle(string path, System.IO.FileMode mode = System.IO.FileMode.Open, System.IO.FileAccess access = System.IO.FileAccess.Read, System.IO.FileShare share = System.IO.FileShare.Read, System.IO.FileOptions options = System.IO.FileOptions.None, long preallocationSize = (long)0) { throw null; }
+        public static Microsoft.Win32.SafeHandles.SafeFileHandle OpenNullHandle() { throw null; }
         public static System.IO.FileStream OpenRead(string path) { throw null; }
         public static System.IO.StreamReader OpenText(string path) { throw null; }
         public static System.IO.FileStream OpenWrite(string path) { throw null; }

src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/File/OpenNullHandle.cs

@@ -0,0 +1,181 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.DotNet.RemoteExecutor;
+using Microsoft.Win32.SafeHandles;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace System.IO.Tests
+{
+    public class File_OpenNullHandle : FileSystemTest
+    {
+        [Fact]
+        public void OpenNullHandle_ReturnsValidHandle()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            Assert.NotNull(handle);
+            Assert.False(handle.IsInvalid);
+            Assert.False(handle.IsClosed);
+        }
+
+        [Fact]
+        public void OpenNullHandle_CanBeUsedWithFileStream()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.ReadWrite);
+            Assert.True(stream.CanRead);
+            Assert.True(stream.CanWrite);
+        }
+
+        [Fact]
+        public void OpenNullHandle_SyncRead_ReturnsZero()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Read);
+            
+            byte[] buffer = new byte[100];
+            int bytesRead = stream.Read(buffer, 0, buffer.Length);
+            Assert.Equal(0, bytesRead);
+        }
+
+        [Fact]
+        public void OpenNullHandle_SyncReadSpan_ReturnsZero()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Read);
+            
+            byte[] buffer = new byte[100];
+            int bytesRead = stream.Read(buffer.AsSpan());
+            Assert.Equal(0, bytesRead);
+        }
+
+        [Fact]
+        public async Task OpenNullHandle_AsyncRead_ReturnsZero()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Read);
+            
+            byte[] buffer = new byte[100];
+            int bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length);
+            Assert.Equal(0, bytesRead);
+        }
+
+        [Fact]
+        public async Task OpenNullHandle_AsyncReadMemory_ReturnsZero()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Read);
+            
+            byte[] buffer = new byte[100];
+            int bytesRead = await stream.ReadAsync(buffer.AsMemory());
+            Assert.Equal(0, bytesRead);
+        }
+
+        [Fact]
+        public void OpenNullHandle_SyncWrite_Succeeds()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Write);
+            
+            byte[] buffer = new byte[100];
+            // Should not throw
+            stream.Write(buffer, 0, buffer.Length);
+        }
+
+        [Fact]
+        public void OpenNullHandle_SyncWriteSpan_Succeeds()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Write);
+            
+            byte[] buffer = new byte[100];
+            // Should not throw
+            stream.Write(buffer.AsSpan());
+        }
+
+        [Fact]
+        public async Task OpenNullHandle_AsyncWrite_Succeeds()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Write);
+            
+            byte[] buffer = new byte[100];
+            // Should not throw
+            await stream.WriteAsync(buffer, 0, buffer.Length);
+        }
+
+        [Fact]
+        public async Task OpenNullHandle_AsyncWriteMemory_Succeeds()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.Write);
+            
+            byte[] buffer = new byte[100];
+            // Should not throw
+            await stream.WriteAsync(buffer.AsMemory());
+        }
+
+        [Fact]
+        public void OpenNullHandle_ReadWriteAccess_BothWork()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.ReadWrite);
+            
+            byte[] writeBuffer = new byte[100];
+            byte[] readBuffer = new byte[100];
+            
+            // Write should succeed
+            stream.Write(writeBuffer, 0, writeBuffer.Length);
+            
+            // Read should return 0
+            int bytesRead = stream.Read(readBuffer, 0, readBuffer.Length);
+            Assert.Equal(0, bytesRead);
+        }
+
+        [Fact]
+        public async Task OpenNullHandle_ReadWriteAccess_BothAsyncOperationsWork()
+        {
+            using SafeFileHandle handle = File.OpenNullHandle();
+            using FileStream stream = new FileStream(handle, FileAccess.ReadWrite);
+            
+            byte[] writeBuffer = new byte[100];
+            byte[] readBuffer = new byte[100];
+            
+            // Write should succeed
+            await stream.WriteAsync(writeBuffer, 0, writeBuffer.Length);
+            
+            // Read should return 0
+            int bytesRead = await stream.ReadAsync(readBuffer, 0, readBuffer.Length);
+            Assert.Equal(0, bytesRead);
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public void OpenNullHandle_DoesNotLockNullDevice()
+        {
+            // Keep a handle open in the current process
+            using SafeFileHandle handle1 = File.OpenNullHandle();
+            using FileStream stream1 = new FileStream(handle1, FileAccess.ReadWrite);
+            
+            // Start a new process that also opens the null device
+            RemoteExecutor.Invoke(() =>
+            {
+                using SafeFileHandle handle2 = File.OpenNullHandle();
+                using FileStream stream2 = new FileStream(handle2, FileAccess.ReadWrite);
+                
+                byte[] buffer = new byte[100];
+                
+                // Both read and write should work in the child process
+                stream2.Write(buffer, 0, buffer.Length);
+                int bytesRead = stream2.Read(buffer, 0, buffer.Length);
+                Assert.Equal(0, bytesRead);
+            }).Dispose();
+            
+            // Original handle should still work after the child process exits
+            byte[] buffer = new byte[100];
+            stream1.Write(buffer, 0, buffer.Length);
+            int bytesRead = stream1.Read(buffer, 0, buffer.Length);
+            Assert.Equal(0, bytesRead);
+        }
+    }
+}

src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/System.IO.FileSystem.Tests.csproj

@@ -226,6 +226,7 @@
     <Compile Include="File\GetSetTimes_String.cs" />
     <Compile Include="File\Open.cs" />
     <Compile Include="File\OpenHandle.cs" />
+    <Compile Include="File\OpenNullHandle.cs" />
     <Compile Include="FileInfo\Create.cs" />
     <Compile Include="FileInfo\CreateText.cs" />
     <Compile Include="FileInfo\Delete.cs" />