Skip to content

Properly handle filesizes larger than 8 Gb #76707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jozkee merged 4 commits into from
Oct 7, 2022
Merged

Properly handle filesizes larger than 8 Gb #76707

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0582aff
Properly handle filesizes larger than 8 Gb
jozkee Oct 6, 2022
7ea5bc6
Add simulated data stream and address feedback
jozkee Oct 7, 2022
c9e4a20
Move tests for 8Gb files to outerloop and disable parallelization
jozkee Oct 7, 2022
354e299
Change TarSizeFieldTooLargeForEntryType on .resx
jozkee Oct 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/libraries/System.Formats.Tar/src/Resources/Strings.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,4 +264,7 @@
<data name="TarEntryFieldExceedsMaxLength" xml:space="preserve">
<value>The field '{0}' exceeds the maximum allowed length for this format.</value>
</data>
<data name="TarSizeFieldTooLargeForEntryFormat" xml:space="preserve">
<value>The value of the size field for the current entry of format '{0}' is beyond the expected length.</value>
</data>
</root>
3 changes: 2 additions & 1 deletion src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,8 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca
return null;
}

long size = (int)TarHelpers.ParseOctal<uint>(buffer.Slice(FieldLocations.Size, FieldLengths.Size));
long size = (long)TarHelpers.ParseOctal<ulong>(buffer.Slice(FieldLocations.Size, FieldLengths.Size));
Debug.Assert(size <= TarHelpers.MaxSizeLength, "size exceeded the max value possible with 11 octal digits. Actual size " + size);
if (size < 0)
{
throw new InvalidDataException(string.Format(SR.TarSizeFieldNegative));
Expand Down
98 changes: 53 additions & 45 deletions src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ internal sealed partial class TarHeader
// Writes the current header as a V7 entry into the archive stream.
internal void WriteAsV7(Stream archiveStream, Span<byte> buffer)
{
long actualLength = WriteV7FieldsToBuffer(buffer);
WriteV7FieldsToBuffer(buffer);

archiveStream.Write(buffer);

if (_dataStream != null)
{
WriteData(archiveStream, _dataStream, actualLength);
WriteData(archiveStream, _dataStream, _size);
}
}

Expand All @@ -43,39 +43,37 @@ internal async Task WriteAsV7Async(Stream archiveStream, Memory<byte> buffer, Ca
{
cancellationToken.ThrowIfCancellationRequested();

long actualLength = WriteV7FieldsToBuffer(buffer.Span);
WriteV7FieldsToBuffer(buffer.Span);

await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);

if (_dataStream != null)
{
await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
}
}

// Writes the V7 header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
private long WriteV7FieldsToBuffer(Span<byte> buffer)
private void WriteV7FieldsToBuffer(Span<byte> buffer)
{
long actualLength = GetTotalDataBytesToWrite();
_size = GetTotalDataBytesToWrite();
TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.V7, _typeFlag);

int tmpChecksum = WriteName(buffer);
tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType);
tmpChecksum += WriteCommonFields(buffer, actualEntryType);
_checksum = WriteChecksum(tmpChecksum, buffer);

return actualLength;
}

// Writes the current header as a Ustar entry into the archive stream.
internal void WriteAsUstar(Stream archiveStream, Span<byte> buffer)
{
long actualLength = WriteUstarFieldsToBuffer(buffer);
WriteUstarFieldsToBuffer(buffer);

archiveStream.Write(buffer);

if (_dataStream != null)
{
WriteData(archiveStream, _dataStream, actualLength);
WriteData(archiveStream, _dataStream, _size);
}
}

Expand All @@ -84,29 +82,27 @@ internal async Task WriteAsUstarAsync(Stream archiveStream, Memory<byte> buffer,
{
cancellationToken.ThrowIfCancellationRequested();

long actualLength = WriteUstarFieldsToBuffer(buffer.Span);
WriteUstarFieldsToBuffer(buffer.Span);

await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);

if (_dataStream != null)
{
await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
}
}

// Writes the Ustar header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
private long WriteUstarFieldsToBuffer(Span<byte> buffer)
private void WriteUstarFieldsToBuffer(Span<byte> buffer)
{
long actualLength = GetTotalDataBytesToWrite();
_size = GetTotalDataBytesToWrite();
TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Ustar, _typeFlag);

int tmpChecksum = WriteUstarName(buffer);
tmpChecksum += WriteCommonFields(buffer, actualLength, actualEntryType);
tmpChecksum += WriteCommonFields(buffer, actualEntryType);
tmpChecksum += WritePosixMagicAndVersion(buffer);
tmpChecksum += WritePosixAndGnuSharedFields(buffer);
_checksum = WriteChecksum(tmpChecksum, buffer);

return actualLength;
}

// Writes the current header as a PAX Global Extended Attributes entry into the archive stream.
Expand Down Expand Up @@ -144,6 +140,7 @@ internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
// First, we write the preceding extended attributes header
TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
// Fill the current header's dict
_size = GetTotalDataBytesToWrite();
CollectExtendedAttributesFromStandardFieldsIfNeeded();
// And pass the attributes to the preceding extended attributes header for writing
extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1);
Expand All @@ -157,12 +154,12 @@ internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
internal async Task WriteAsPaxAsync(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
{
Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);

cancellationToken.ThrowIfCancellationRequested();

// First, we write the preceding extended attributes header
TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
// Fill the current header's dict
_size = GetTotalDataBytesToWrite();
CollectExtendedAttributesFromStandardFieldsIfNeeded();
// And pass the attributes to the preceding extended attributes header for writing
await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
Expand Down Expand Up @@ -243,13 +240,13 @@ private static TarHeader GetGnuLongMetadataHeader(TarEntryType entryType, string
// Writes the current header as a GNU entry into the archive stream.
internal void WriteAsGnuInternal(Stream archiveStream, Span<byte> buffer)
{
WriteAsGnuSharedInternal(buffer, out long actualLength);
WriteAsGnuSharedInternal(buffer);

archiveStream.Write(buffer);

if (_dataStream != null)
{
WriteData(archiveStream, _dataStream, actualLength);
WriteData(archiveStream, _dataStream, _size);
}
}

Expand All @@ -258,23 +255,23 @@ internal async Task WriteAsGnuInternalAsync(Stream archiveStream, Memory<byte> b
{
cancellationToken.ThrowIfCancellationRequested();

WriteAsGnuSharedInternal(buffer.Span, out long actualLength);
WriteAsGnuSharedInternal(buffer.Span);

await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);

if (_dataStream != null)
{
await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
}
}

// Shared checksum and data length calculations for GNU entry writing.
private void WriteAsGnuSharedInternal(Span<byte> buffer, out long actualLength)
private void WriteAsGnuSharedInternal(Span<byte> buffer)
{
actualLength = GetTotalDataBytesToWrite();
_size = GetTotalDataBytesToWrite();

int tmpChecksum = WriteName(buffer);
tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag));
tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag));
tmpChecksum += WriteGnuMagicAndVersion(buffer);
tmpChecksum += WritePosixAndGnuSharedFields(buffer);
tmpChecksum += WriteGnuFields(buffer);
Expand All @@ -285,45 +282,44 @@ private void WriteAsGnuSharedInternal(Span<byte> buffer, out long actualLength)
// Writes the current header as a PAX Extended Attributes entry into the archive stream.
private void WriteAsPaxExtendedAttributes(Stream archiveStream, Span<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber)
{
WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber);
_dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
WriteAsPaxInternal(archiveStream, buffer);
}

// Asynchronously writes the current header as a PAX Extended Attributes entry into the archive stream and returns the value of the final checksum.
private Task WriteAsPaxExtendedAttributesAsync(Stream archiveStream, Memory<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();

WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber);
_dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
return WriteAsPaxInternalAsync(archiveStream, buffer, cancellationToken);
}

// Initializes the name, mode and type flag of a PAX extended attributes entry.
private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber)
private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber, Dictionary<string, string> extendedAttributes)
{
Debug.Assert(isGea && globalExtendedAttributesEntryNumber >= 0 || !isGea && globalExtendedAttributesEntryNumber < 0);

_dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
_name = isGea ?
GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber) :
GenerateExtendedAttributeName();

_mode = TarHelpers.GetDefaultMode(_typeFlag);
_size = GetTotalDataBytesToWrite();
_typeFlag = isGea ? TarEntryType.GlobalExtendedAttributes : TarEntryType.ExtendedAttributes;
}

// Both the Extended Attributes and Global Extended Attributes entry headers are written in a similar way, just the data changes
// This method writes an entry as both entries require, using the data from the current header instance.
private void WriteAsPaxInternal(Stream archiveStream, Span<byte> buffer)
{
WriteAsPaxSharedInternal(buffer, out long actualLength);
WriteAsPaxSharedInternal(buffer);

archiveStream.Write(buffer);

if (_dataStream != null)
{
WriteData(archiveStream, _dataStream, actualLength);
WriteData(archiveStream, _dataStream, _size);
}
}

Expand All @@ -333,23 +329,21 @@ private async Task WriteAsPaxInternalAsync(Stream archiveStream, Memory<byte> bu
{
cancellationToken.ThrowIfCancellationRequested();

WriteAsPaxSharedInternal(buffer.Span, out long actualLength);
WriteAsPaxSharedInternal(buffer.Span);

await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);

if (_dataStream != null)
{
await WriteDataAsync(archiveStream, _dataStream, actualLength, cancellationToken).ConfigureAwait(false);
await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
}
}

// Shared checksum and data length calculations for PAX entry writing.
private void WriteAsPaxSharedInternal(Span<byte> buffer, out long actualLength)
private void WriteAsPaxSharedInternal(Span<byte> buffer)
{
actualLength = GetTotalDataBytesToWrite();

int tmpChecksum = WriteName(buffer);
tmpChecksum += WriteCommonFields(buffer, actualLength, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag));
tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag));
tmpChecksum += WritePosixMagicAndVersion(buffer);
tmpChecksum += WritePosixAndGnuSharedFields(buffer);

Expand Down Expand Up @@ -446,7 +440,7 @@ private int WriteUstarName(Span<byte> buffer)
}

// Writes all the common fields shared by all formats into the specified spans.
private int WriteCommonFields(Span<byte> buffer, long actualLength, TarEntryType actualEntryType)
private int WriteCommonFields(Span<byte> buffer, TarEntryType actualEntryType)
{
// Don't write an empty LinkName if the entry is a hardlink or symlink
Debug.Assert(!string.IsNullOrEmpty(_linkName) ^ (_typeFlag is not TarEntryType.SymbolicLink and not TarEntryType.HardLink));
Expand All @@ -468,11 +462,21 @@ private int WriteCommonFields(Span<byte> buffer, long actualLength, TarEntryType
checksum += FormatOctal(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid));
}

_size = actualLength;

if (_size > 0)
{
checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size));
if (_size <= TarHelpers.MaxSizeLength)
{
checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size));
}
else if (_format is not TarEntryFormat.Pax)
{
throw new ArgumentException(SR.Format(SR.TarSizeFieldTooLargeForEntryFormat, _format));
}
else
{
Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes);
Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength);
}
}

checksum += WriteAsTimestamp(_mTime, buffer.Slice(FieldLocations.MTime, FieldLengths.MTime));
Expand Down Expand Up @@ -732,10 +736,14 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded()
ExtendedAttributes[PaxEaLinkName] = _linkName;
}

if (_size > 99_999_999)
if (_size > TarHelpers.MaxSizeLength)
{
ExtendedAttributes[PaxEaSize] = _size.ToString();
}
else
{
ExtendedAttributes.Remove(PaxEaSize);
}

// Sets the specified string to the dictionary if it's longer than the specified max byte length; otherwise, remove it.
static void TryAddStringField(Dictionary<string, string> extendedAttributes, string key, string? value, int maxLength)
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ internal static partial class TarHelpers
{
internal const short RecordSize = 512;
internal const int MaxBufferLength = 4096;
internal const long MaxSizeLength = (1L << 33) - 1;

// Default mode for TarEntry created for a file-type.
private const UnixFileMode DefaultFileMode =
Expand Down
1 change: 1 addition & 0 deletions src/libraries/System.Formats.Tar/tests/TarTestsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ public abstract partial class TarTestsBase : FileCleanupTestBase
internal const string FourBytesCharacter = "\uD83D\uDE12";
internal const char Separator = '/';
internal const int MaxPathComponent = 255;
internal const long LegacyMaxFileSize = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb.

private static readonly string[] V7TestCaseNames = new[]
{
Expand Down
Loading