Trim file type bits from mode header (#77151)

* Trim file type bits from mode header

* Define and use ValidUnixFileModes

* Rev System.Formats.Tar.TestData version

* Move ValidUnixFileModes to shared helpers

Author: am11

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs

@@ -138,10 +138,12 @@ public string LinkName
         /// <remarks>The value in this field has no effect on Windows platforms.</remarks>
         public UnixFileMode Mode
         {
-            get => (UnixFileMode)_header._mode;
+            // Some paths do not use the setter, and we want to return valid UnixFileMode.
+            // This mask only keeps the least significant 12 bits.
+            get => (UnixFileMode)(_header._mode & (int)TarHelpers.ValidUnixFileModes);
             set
             {
-                if ((int)value is < 0 or > 4095) // 4095 in decimal is 7777 in octal
+                if ((value & ~TarHelpers.ValidUnixFileModes) != 0) // throw on invalid UnixFileModes
                 {
                     throw new ArgumentOutOfRangeException(nameof(value));
                 }

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs

@@ -21,6 +21,20 @@ internal static partial class TarHelpers
         internal const int MaxBufferLength = 4096;
         internal const long MaxSizeLength = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb.
 
+        internal const UnixFileMode ValidUnixFileModes =
+            UnixFileMode.UserRead |
+            UnixFileMode.UserWrite |
+            UnixFileMode.UserExecute |
+            UnixFileMode.GroupRead |
+            UnixFileMode.GroupWrite |
+            UnixFileMode.GroupExecute |
+            UnixFileMode.OtherRead |
+            UnixFileMode.OtherWrite |
+            UnixFileMode.OtherExecute |
+            UnixFileMode.StickyBit |
+            UnixFileMode.SetGroup |
+            UnixFileMode.SetUser;
+
         // Default mode for TarEntry created for a file-type.
         private const UnixFileMode DefaultFileMode =
             UnixFileMode.UserRead | UnixFileMode.UserWrite |

src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.Unix.cs

@@ -66,7 +66,8 @@ private TarEntry ConstructEntryForWriting(string fullPath, string entryName, Fil
             entry._header._aTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(status.ATime);
             entry._header._cTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(status.CTime);
 
-            entry._header._mode = status.Mode & 4095; // First 12 bits
+            // This mask only keeps the least significant 12 bits valid for UnixFileModes
+            entry._header._mode = status.Mode & (int)TarHelpers.ValidUnixFileModes;
 
             // Uid and UName
             entry._header._uid = (int)status.Uid;

src/libraries/System.Formats.Tar/tests/TarFile/TarFile.ExtractToDirectoryAsync.Stream.Tests.cs

@@ -97,6 +97,19 @@ public async Task ExtractEntry_ManySubfolderSegments_NoPrecedingDirectoryEntries
             }
         }
 
+        [Fact]
+        public async Task ExtractEntry_DockerImageTarWithFileTypeInDirectoriesInMode_SuccessfullyExtracts_Async()
+        {
+            using (TempDirectory root = new TempDirectory())
+            {
+                await using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", "docker-hello-world");
+                await TarFile.ExtractToDirectoryAsync(archiveStream, root.Path, overwriteFiles: true);
+
+                Assert.True(File.Exists(Path.Join(root.Path, "manifest.json")));
+                Assert.True(File.Exists(Path.Join(root.Path, "repositories")));
+            }
+        }
+
         [Theory]
         [InlineData(TarEntryType.SymbolicLink)]
         [InlineData(TarEntryType.HardLink)]