From 3d29c183efba9268aab608af8fb4fc575043a7bc Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 28 Oct 2025 15:00:11 +0000 Subject: [PATCH 01/25] basic async usage --- .../Archives/Rar/RarArchiveEntry.cs | 7 +- .../Rar/MultiVolumeReadOnlyStream.cs | 130 ++++++++++++++++++ .../Compressors/Rar/RarBLAKE2spStream.cs | 55 ++++++++ .../Compressors/Rar/RarCrcStream.cs | 51 +++++++ .../Compressors/Rar/RarStream.cs | 32 +++++ 5 files changed, 273 insertions(+), 2 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 18a879695..74a3ce4c2 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -86,8 +86,11 @@ public Stream OpenEntryStream() ); } - public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => - Task.FromResult(OpenEntryStream()); + public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(OpenEntryStream()); + } public bool IsComplete { diff --git a/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs b/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs index b96533ce3..d20fecd07 100644 --- a/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs +++ b/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs @@ -150,6 +150,136 @@ public override int Read(byte[] buffer, int offset, int count) return totalRead; } + public override async System.Threading.Tasks.Task ReadAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + var totalRead = 0; + var currentOffset = offset; + var currentCount = count; + while (currentCount > 0) + { + var readSize = currentCount; + if (currentCount > maxPosition - currentPosition) + { + readSize = (int)(maxPosition - currentPosition); + } + + var read = await currentStream + .ReadAsync(buffer, currentOffset, readSize, cancellationToken) + .ConfigureAwait(false); + if (read < 0) + { + throw new EndOfStreamException(); + } + + currentPosition += read; + currentOffset += read; + currentCount -= read; + totalRead += read; + if ( + ((maxPosition - currentPosition) == 0) + && filePartEnumerator.Current.FileHeader.IsSplitAfter + ) + { + if (filePartEnumerator.Current.FileHeader.R4Salt != null) + { + throw new InvalidFormatException( + "Sharpcompress currently does not support multi-volume decryption." + ); + } + var fileName = filePartEnumerator.Current.FileHeader.FileName; + if (!filePartEnumerator.MoveNext()) + { + throw new InvalidFormatException( + "Multi-part rar file is incomplete. Entry expects a new volume: " + + fileName + ); + } + InitializeNextFilePart(); + } + else + { + break; + } + } + currentPartTotalReadBytes += totalRead; + currentEntryTotalReadBytes += totalRead; + streamListener.FireCompressedBytesRead( + currentPartTotalReadBytes, + currentEntryTotalReadBytes + ); + return totalRead; + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override async System.Threading.Tasks.ValueTask ReadAsync( + Memory buffer, + System.Threading.CancellationToken cancellationToken = default + ) + { + var totalRead = 0; + var currentOffset = 0; + var currentCount = buffer.Length; + while (currentCount > 0) + { + var readSize = currentCount; + if (currentCount > maxPosition - currentPosition) + { + readSize = (int)(maxPosition - currentPosition); + } + + var read = await currentStream + .ReadAsync(buffer.Slice(currentOffset, readSize), cancellationToken) + .ConfigureAwait(false); + if (read < 0) + { + throw new EndOfStreamException(); + } + + currentPosition += read; + currentOffset += read; + currentCount -= read; + totalRead += read; + if ( + ((maxPosition - currentPosition) == 0) + && filePartEnumerator.Current.FileHeader.IsSplitAfter + ) + { + if (filePartEnumerator.Current.FileHeader.R4Salt != null) + { + throw new InvalidFormatException( + "Sharpcompress currently does not support multi-volume decryption." + ); + } + var fileName = filePartEnumerator.Current.FileHeader.FileName; + if (!filePartEnumerator.MoveNext()) + { + throw new InvalidFormatException( + "Multi-part rar file is incomplete. Entry expects a new volume: " + + fileName + ); + } + InitializeNextFilePart(); + } + else + { + break; + } + } + currentPartTotalReadBytes += totalRead; + currentEntryTotalReadBytes += totalRead; + streamListener.FireCompressedBytesRead( + currentPartTotalReadBytes, + currentEntryTotalReadBytes + ); + return totalRead; + } +#endif + public override bool CanRead => true; public override bool CanSeek => false; diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs index c0aead0db..5de6bf032 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -333,4 +333,59 @@ public override int Read(byte[] buffer, int offset, int count) return result; } + + public override async System.Threading.Tasks.Task ReadAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + var result = await base.ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + if (result != 0) + { + Update(_blake2sp, new ReadOnlySpan(buffer, offset, result), result); + } + else + { + _hash = Final(_blake2sp); + if (!disableCRCCheck && !(GetCrc().SequenceEqual(readStream.CurrentCrc)) && count != 0) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + } + + return result; + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override async System.Threading.Tasks.ValueTask ReadAsync( + Memory buffer, + System.Threading.CancellationToken cancellationToken = default + ) + { + var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (result != 0) + { + Update(_blake2sp, buffer.Span.Slice(0, result), result); + } + else + { + _hash = Final(_blake2sp); + if ( + !disableCRCCheck + && !(GetCrc().SequenceEqual(readStream.CurrentCrc)) + && buffer.Length != 0 + ) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + } + + return result; + } +#endif } diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 301910359..5473370f4 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -77,4 +77,55 @@ public override int Read(byte[] buffer, int offset, int count) return result; } + + public override async System.Threading.Tasks.Task ReadAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + var result = await base.ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + if (result != 0) + { + currentCrc = RarCRC.CheckCrc(currentCrc, buffer, offset, result); + } + else if ( + !disableCRC + && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) + && count != 0 + ) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + + return result; + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override async System.Threading.Tasks.ValueTask ReadAsync( + Memory buffer, + System.Threading.CancellationToken cancellationToken = default + ) + { + var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (result != 0) + { + currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span.ToArray(), 0, result); + } + else if ( + !disableCRC + && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) + && buffer.Length != 0 + ) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + + return result; + } +#endif } diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index f2f4b2198..d44cf948b 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -131,6 +131,38 @@ public override int Read(byte[] buffer, int offset, int count) return outTotal; } + public override System.Threading.Tasks.Task ReadAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + return System.Threading.Tasks.Task.FromResult(Read(buffer, offset, count)); + } + +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + public override System.Threading.Tasks.ValueTask ReadAsync( + Memory buffer, + System.Threading.CancellationToken cancellationToken = default + ) + { + cancellationToken.ThrowIfCancellationRequested(); + var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); + try + { + var bytesRead = Read(array, 0, buffer.Length); + new ReadOnlySpan(array, 0, bytesRead).CopyTo(buffer.Span); + return new System.Threading.Tasks.ValueTask(bytesRead); + } + finally + { + System.Buffers.ArrayPool.Shared.Return(array); + } + } +#endif + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); From 1ba529a9d5586a90ef68a0fdf50d7d9c2495aa03 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 28 Oct 2025 15:29:41 +0000 Subject: [PATCH 02/25] first pass of async files --- .../Compressors/Rar/IRarUnpack.cs | 5 + .../Compressors/Rar/RarStream.cs | 33 +- .../Compressors/Rar/UnpackV1/Unpack.cs | 77 +++ .../Compressors/Rar/UnpackV1/Unpack15.cs | 34 ++ .../Compressors/Rar/UnpackV1/Unpack50.cs | 457 +++++++++++++----- .../Compressors/Rar/UnpackV2017/Unpack.cs | 63 +++ 6 files changed, 557 insertions(+), 112 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs index 651e2c8ac..03f3cd298 100644 --- a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs +++ b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs @@ -1,4 +1,6 @@ using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; namespace SharpCompress.Compressors.Rar; @@ -8,6 +10,9 @@ internal interface IRarUnpack void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStream); void DoUnpack(); + Task DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream, CancellationToken cancellationToken); + Task DoUnpackAsync(CancellationToken cancellationToken); + // eg u/i pause/resume button bool Suspended { get; set; } diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index d44cf948b..43d6ff4ae 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -131,15 +131,42 @@ public override int Read(byte[] buffer, int offset, int count) return outTotal; } - public override System.Threading.Tasks.Task ReadAsync( + public override async System.Threading.Tasks.Task ReadAsync( byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken ) { - cancellationToken.ThrowIfCancellationRequested(); - return System.Threading.Tasks.Task.FromResult(Read(buffer, offset, count)); + outTotal = 0; + if (tmpCount > 0) + { + var toCopy = tmpCount < count ? tmpCount : count; + Buffer.BlockCopy(tmpBuffer, tmpOffset, buffer, offset, toCopy); + tmpOffset += toCopy; + tmpCount -= toCopy; + offset += toCopy; + count -= toCopy; + outTotal += toCopy; + } + if (count > 0 && unpack.DestSize > 0) + { + outBuffer = buffer; + outOffset = offset; + outCount = count; + fetch = true; + await unpack.DoUnpackAsync(cancellationToken).ConfigureAwait(false); + fetch = false; + } + _position += outTotal; + if (count > 0 && outTotal == 0 && _position != Length) + { + // sanity check, eg if we try to decompress a redir entry + throw new InvalidOperationException( + $"unpacked file size does not match header: expected {Length} found {_position}" + ); + } + return outTotal; } #if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index cf7b36013..c75fbb6a5 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -155,6 +155,25 @@ public void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStrea DoUnpack(); } + public async System.Threading.Tasks.Task DoUnpackAsync( + FileHeader fileHeader, + Stream readStream, + Stream writeStream, + System.Threading.CancellationToken cancellationToken = default + ) + { + destUnpSize = fileHeader.UncompressedSize; + this.fileHeader = fileHeader; + this.readStream = readStream; + this.writeStream = writeStream; + if (!fileHeader.IsSolid) + { + Init(null); + } + suspended = false; + await DoUnpackAsync(cancellationToken).ConfigureAwait(false); + } + public void DoUnpack() { if (fileHeader.CompressionMethod == 0) @@ -189,6 +208,44 @@ public void DoUnpack() } } + public async System.Threading.Tasks.Task DoUnpackAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + if (fileHeader.CompressionMethod == 0) + { + await UnstoreFileAsync(cancellationToken).ConfigureAwait(false); + return; + } + // TODO: When compression methods are converted to async, call them here + // For now, fall back to synchronous version + switch (fileHeader.CompressionAlgorithm) + { + case 15: // rar 1.5 compression + unpack15(fileHeader.IsSolid); + break; + + case 20: // rar 2.x compression + case 26: // files larger than 2GB + unpack20(fileHeader.IsSolid); + break; + + case 29: // rar 3.x compression + case 36: // alternative hash + Unpack29(fileHeader.IsSolid); + break; + + case 50: // rar 5.x compression + await Unpack5Async(fileHeader.IsSolid,cancellationToken).ConfigureAwait(false); + break; + + default: + throw new InvalidFormatException( + "unknown rar compression version " + fileHeader.CompressionAlgorithm + ); + } + } + private void UnstoreFile() { Span buffer = stackalloc byte[(int)Math.Min(0x10000, destUnpSize)]; @@ -205,6 +262,26 @@ private void UnstoreFile() } while (!suspended && destUnpSize > 0); } + private async System.Threading.Tasks.Task UnstoreFileAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var buffer = new byte[(int)Math.Min(0x10000, destUnpSize)]; + do + { + var code = await readStream + .ReadAsync(buffer, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); + if (code == 0 || code == -1) + { + break; + } + code = code < destUnpSize ? code : (int)destUnpSize; + await writeStream.WriteAsync(buffer, 0, code, cancellationToken).ConfigureAwait(false); + destUnpSize -= code; + } while (!suspended && destUnpSize > 0); + } + private void Unpack29(bool solid) { Span DDecode = stackalloc int[PackDef.DC]; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs index 96c9ac6f3..ac962b67c 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs @@ -351,6 +351,40 @@ private bool unpReadBuf() return (readCode != -1); } + private async System.Threading.Tasks.Task unpReadBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var dataSize = readTop - inAddr; + if (dataSize < 0) + { + return (false); + } + if (inAddr > MAX_SIZE / 2) + { + if (dataSize > 0) + { + Array.Copy(InBuf, inAddr, InBuf, 0, dataSize); + } + inAddr = 0; + readTop = dataSize; + } + else + { + dataSize = readTop; + } + + var readCode = await readStream + .ReadAsync(InBuf, dataSize, (MAX_SIZE - dataSize) & ~0xf, cancellationToken) + .ConfigureAwait(false); + if (readCode > 0) + { + readTop += readCode; + } + readBorder = readTop - 30; + return (readCode != -1); + } + private int getShortLen1(int pos) => pos == 1 ? Buf60 + 3 : ShortLen1[pos]; private int getShortLen2(int pos) => pos == 3 ? Buf60 + 3 : ShortLen2[pos]; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs index f251e47d9..39765e80b 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs @@ -479,6 +479,354 @@ private bool UnpReadBuf() return ReadCode != -1; } + private async System.Threading.Tasks.Task UnpReadBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var DataSize = ReadTop - Inp.InAddr; // Data left to process. + if (DataSize < 0) + { + return false; + } + + BlockHeader.BlockSize -= Inp.InAddr - BlockHeader.BlockStart; + if (Inp.InAddr > MAX_SIZE / 2) + { + if (DataSize > 0) + { + Array.Copy(InBuf, inAddr, InBuf, 0, DataSize); + } + + Inp.InAddr = 0; + ReadTop = DataSize; + } + else + { + DataSize = ReadTop; + } + + var ReadCode = 0; + if (MAX_SIZE != DataSize) + { + ReadCode = await readStream + .ReadAsync(InBuf, DataSize, MAX_SIZE - DataSize, cancellationToken) + .ConfigureAwait(false); + } + + if (ReadCode > 0) // Can be also -1. + { + ReadTop += ReadCode; + } + + ReadBorder = ReadTop - 30; + BlockHeader.BlockStart = Inp.InAddr; + if (BlockHeader.BlockSize != -1) // '-1' means not defined yet. + { + ReadBorder = Math.Min(ReadBorder, BlockHeader.BlockStart + BlockHeader.BlockSize - 1); + } + return ReadCode != -1; + } + + public async System.Threading.Tasks.Task Unpack5Async( + bool Solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + FileExtracted = true; + + if (!Suspended) + { + UnpInitData(Solid); + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + // Check TablesRead5 to be sure that we read tables at least once + // regardless of current block header TablePresent flag. + // So we can safefly use these tables below. + if ( + !await ReadBlockHeaderAsync(cancellationToken).ConfigureAwait(false) + || !ReadTables() + || !TablesRead5 + ) + { + return; + } + } + + while (true) + { + UnpPtr &= MaxWinMask; + + if (Inp.InAddr >= ReadBorder) + { + var FileDone = false; + + // We use 'while', because for empty block containing only Huffman table, + // we'll be on the block border once again just after reading the table. + while ( + Inp.InAddr > BlockHeader.BlockStart + BlockHeader.BlockSize - 1 + || Inp.InAddr == BlockHeader.BlockStart + BlockHeader.BlockSize - 1 + && Inp.InBit >= BlockHeader.BlockBitSize + ) + { + if (BlockHeader.LastBlockInFile) + { + FileDone = true; + break; + } + if ( + !await ReadBlockHeaderAsync(cancellationToken).ConfigureAwait(false) + || !ReadTables() + ) + { + return; + } + } + if (FileDone || !await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + break; + } + } + + if ( + ((WriteBorder - UnpPtr) & MaxWinMask) < PackDef.MAX_LZ_MATCH + 3 + && WriteBorder != UnpPtr + ) + { + UnpWriteBuf(); + if (WrittenFileSize > DestUnpSize) + { + return; + } + + if (Suspended) + { + FileExtracted = false; + return; + } + } + + //uint MainSlot=DecodeNumber(Inp,LD); + var MainSlot = this.DecodeNumber(LD); + if (MainSlot < 256) + { + // if (Fragmented) + // FragWindow[UnpPtr++]=(byte)MainSlot; + // else + Window[UnpPtr++] = (byte)MainSlot; + continue; + } + if (MainSlot >= 262) + { + var Length = SlotToLength(MainSlot - 262); + + //uint DBits,Distance=1,DistSlot=DecodeNumber(Inp,&BlockTables.DD); + int DBits; + uint Distance = 1, + DistSlot = this.DecodeNumber(DD); + if (DistSlot < 4) + { + DBits = 0; + Distance += DistSlot; + } + else + { + //DBits=DistSlot/2 - 1; + DBits = (int)((DistSlot / 2) - 1); + Distance += (2 | (DistSlot & 1)) << DBits; + } + + if (DBits > 0) + { + if (DBits >= 4) + { + if (DBits > 4) + { + Distance += ((Inp.getbits() >> (36 - DBits)) << 4); + Inp.AddBits(DBits - 4); + } + //uint LowDist=DecodeNumber(Inp,&BlockTables.LDD); + var LowDist = this.DecodeNumber(LDD); + Distance += LowDist; + } + else + { + Distance += Inp.getbits() >> (32 - DBits); + Inp.AddBits(DBits); + } + } + + if (Distance > 0x100) + { + Length++; + if (Distance > 0x2000) + { + Length++; + if (Distance > 0x40000) + { + Length++; + } + } + } + + InsertOldDist(Distance); + LastLength = Length; + // if (Fragmented) + // FragWindow.CopyString(Length,Distance,UnpPtr,MaxWinMask); + // else + CopyString(Length, Distance); + continue; + } + if (MainSlot == 256) + { + var Filter = new UnpackFilter(); + if ( + !await ReadFilterAsync(Filter, cancellationToken).ConfigureAwait(false) + || !AddFilter(Filter) + ) + { + break; + } + + continue; + } + if (MainSlot == 257) + { + if (LastLength != 0) + // if (Fragmented) + // FragWindow.CopyString(LastLength,OldDist[0],UnpPtr,MaxWinMask); + // else + //CopyString(LastLength,OldDist[0]); + { + CopyString(LastLength, OldDistN(0)); + } + + continue; + } + if (MainSlot < 262) + { + //uint DistNum=MainSlot-258; + var DistNum = (int)(MainSlot - 258); + //uint Distance=OldDist[DistNum]; + var Distance = OldDistN(DistNum); + //for (uint I=DistNum;I>0;I--) + for (var I = DistNum; I > 0; I--) + //OldDistN[I]=OldDistN(I-1); + { + SetOldDistN(I, OldDistN(I - 1)); + } + + //OldDistN[0]=Distance; + SetOldDistN(0, Distance); + + var LengthSlot = this.DecodeNumber(RD); + var Length = SlotToLength(LengthSlot); + LastLength = Length; + // if (Fragmented) + // FragWindow.CopyString(Length,Distance,UnpPtr,MaxWinMask); + // else + CopyString(Length, Distance); + continue; + } + } + UnpWriteBuf(); + } + + private async System.Threading.Tasks.Task ReadBlockHeaderAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + Header.HeaderSize = 0; + + if (!Inp.ExternalBuffer && Inp.InAddr > ReadTop - 7) + { + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return false; + } + } + + //Inp.faddbits((8-Inp.InBit)&7); + Inp.faddbits((uint)((8 - Inp.InBit) & 7)); + + var BlockFlags = (byte)(Inp.fgetbits() >> 8); + Inp.faddbits(8); + //uint ByteCount=((BlockFlags>>3)&3)+1; // Block size byte count. + var ByteCount = (uint)(((BlockFlags >> 3) & 3) + 1); // Block size byte count. + + if (ByteCount == 4) + { + return false; + } + + //Header.HeaderSize=2+ByteCount; + Header.HeaderSize = (int)(2 + ByteCount); + + Header.BlockBitSize = (BlockFlags & 7) + 1; + + var SavedCheckSum = (byte)(Inp.fgetbits() >> 8); + Inp.faddbits(8); + + var BlockSize = 0; + //for (uint I=0;I>8)<<(I*8); + BlockSize += (int)(Inp.fgetbits() >> 8) << (I * 8); + Inp.AddBits(8); + } + + Header.BlockSize = BlockSize; + var CheckSum = (byte)(0x5a ^ BlockFlags ^ BlockSize ^ (BlockSize >> 8) ^ (BlockSize >> 16)); + if (CheckSum != SavedCheckSum) + { + return false; + } + + Header.BlockStart = Inp.InAddr; + ReadBorder = Math.Min(ReadBorder, Header.BlockStart + Header.BlockSize - 1); + + Header.LastBlockInFile = (BlockFlags & 0x40) != 0; + Header.TablePresent = (BlockFlags & 0x80) != 0; + return true; + } + + private async System.Threading.Tasks.Task ReadFilterAsync( + UnpackFilter Filter, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (!Inp.ExternalBuffer && Inp.InAddr > ReadTop - 16) + { + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return false; + } + } + + Filter.uBlockStart = ReadFilterData(); + Filter.uBlockLength = ReadFilterData(); + if (Filter.BlockLength > MAX_FILTER_BLOCK_SIZE) + { + Filter.BlockLength = 0; + } + + //Filter.Type=Inp.fgetbits()>>13; + Filter.Type = (byte)(Inp.fgetbits() >> 13); + Inp.faddbits(3); + + if (Filter.Type == (byte)FilterType.FILTER_DELTA) + { + //Filter.Channels=(Inp.fgetbits()>>11)+1; + Filter.Channels = (byte)((Inp.fgetbits() >> 11) + 1); + Inp.faddbits(5); + } + + return true; + } + //? // void UnpWriteBuf() // { @@ -815,115 +1163,6 @@ private bool ReadBlockHeader() return true; } - //? - // bool ReadTables(BitInput Inp, ref UnpackBlockHeader Header, ref UnpackBlockTables Tables) - // { - // if (!Header.TablePresent) - // return true; - // - // if (!Inp.ExternalBuffer && Inp.InAddr>ReadTop-25) - // if (!UnpReadBuf()) - // return false; - // - // byte BitLength[BC]; - // for (uint I=0;I> 12); - // Inp.faddbits(4); - // if (Length==15) - // { - // uint ZeroCount=(byte)(Inp.fgetbits() >> 12); - // Inp.faddbits(4); - // if (ZeroCount==0) - // BitLength[I]=15; - // else - // { - // ZeroCount+=2; - // while (ZeroCount-- > 0 && IReadTop-5) - // if (!UnpReadBuf()) - // return false; - // uint Number=DecodeNumber(Inp,&Tables.BD); - // if (Number<16) - // { - // Table[I]=Number; - // I++; - // } - // else - // if (Number<18) - // { - // uint N; - // if (Number==16) - // { - // N=(Inp.fgetbits() >> 13)+3; - // Inp.faddbits(3); - // } - // else - // { - // N=(Inp.fgetbits() >> 9)+11; - // Inp.faddbits(7); - // } - // if (I==0) - // { - // // We cannot have "repeat previous" code at the first position. - // // Multiple such codes would shift Inp position without changing I, - // // which can lead to reading beyond of Inp boundary in mutithreading - // // mode, where Inp.ExternalBuffer disables bounds check and we just - // // reserve a lot of buffer space to not need such check normally. - // return false; - // } - // else - // while (N-- > 0 && I> 13)+3; - // Inp.faddbits(3); - // } - // else - // { - // N=(Inp.fgetbits() >> 9)+11; - // Inp.faddbits(7); - // } - // while (N-- > 0 && IReadTop) - // return false; - // MakeDecodeTables(&Table[0],&Tables.LD,NC); - // MakeDecodeTables(&Table[NC],&Tables.DD,DC); - // MakeDecodeTables(&Table[NC+DC],&Tables.LDD,LDC); - // MakeDecodeTables(&Table[NC+DC+LDC],&Tables.RD,RC); - // return true; - // } - //? - // void InitFilters() - // { - // Filters.SoftReset(); - // } } #endif diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index 22fdc7463..a66f0acbe 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -29,6 +29,15 @@ private int UnpIO_UnpRead(byte[] buf, int offset, int count) => // NOTE: caller has logic to check for -1 for error we throw instead. readStream.Read(buf, offset, count); + private async System.Threading.Tasks.Task UnpIO_UnpReadAsync( + byte[] buf, + int offset, + int count, + System.Threading.CancellationToken cancellationToken = default + ) => + // NOTE: caller has logic to check for -1 for error we throw instead. + await readStream.ReadAsync(buf, offset, count, cancellationToken).ConfigureAwait(false); + private void UnpIO_UnpWrite(byte[] buf, size_t offset, uint count) => writeStream.Write(buf, checked((int)offset), checked((int)count)); @@ -53,6 +62,25 @@ public void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStrea DoUnpack(); } + public async System.Threading.Tasks.Task DoUnpackAsync( + FileHeader fileHeader, + Stream readStream, + Stream writeStream, + System.Threading.CancellationToken cancellationToken = default + ) + { + DestUnpSize = fileHeader.UncompressedSize; + this.fileHeader = fileHeader; + this.readStream = readStream; + this.writeStream = writeStream; + if (!fileHeader.IsStored) + { + Init(fileHeader.WindowSize, fileHeader.IsSolid); + } + Suspended = false; + await DoUnpackAsync(cancellationToken).ConfigureAwait(false); + } + public void DoUnpack() { if (fileHeader.IsStored) @@ -65,6 +93,22 @@ public void DoUnpack() } } + public async System.Threading.Tasks.Task DoUnpackAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + if (fileHeader.IsStored) + { + await UnstoreFileAsync(cancellationToken).ConfigureAwait(false); + } + else + { + // TODO: When compression methods are converted to async, call them here + // For now, fall back to synchronous version + DoUnpack(fileHeader.CompressionAlgorithm, fileHeader.IsSolid); + } + } + private void UnstoreFile() { Span b = stackalloc byte[(int)Math.Min(0x10000, DestUnpSize)]; @@ -80,6 +124,25 @@ private void UnstoreFile() } while (!Suspended); } + private async System.Threading.Tasks.Task UnstoreFileAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var buffer = new byte[(int)Math.Min(0x10000, DestUnpSize)]; + do + { + var n = await readStream + .ReadAsync(buffer, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); + if (n == 0) + { + break; + } + await writeStream.WriteAsync(buffer, 0, n, cancellationToken).ConfigureAwait(false); + DestUnpSize -= n; + } while (!Suspended); + } + public bool Suspended { get; set; } public long DestSize => DestUnpSize; From b23f031db9ed7dfe96951307459ef5249fab231f Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 28 Oct 2025 15:52:18 +0000 Subject: [PATCH 03/25] add async reads --- .../Archives/Rar/RarArchiveEntry.cs | 28 +++++++++++++++---- .../Compressors/Rar/RarCrcStream.cs | 24 +++++++++++++++- .../Compressors/Rar/RarStream.cs | 13 +++++++++ src/SharpCompress/Readers/Rar/RarReader.cs | 4 +-- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 74a3ce4c2..b795d1e68 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -70,26 +70,44 @@ public override long CompressedSize public Stream OpenEntryStream() { + RarStream stream; if (IsRarV3) { - return new RarStream( + stream = new RarStream( archive.UnpackV1.Value, FileHeader, new MultiVolumeReadOnlyStream(Parts.Cast(), archive) ); } - return new RarStream( + stream = new RarStream( archive.UnpackV2017.Value, FileHeader, new MultiVolumeReadOnlyStream(Parts.Cast(), archive) ); + stream.Initialize(); + return stream; } - public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) + public async Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) { - cancellationToken.ThrowIfCancellationRequested(); - return Task.FromResult(OpenEntryStream()); + RarStream stream; + if (IsRarV3) + { + stream = new RarStream( + archive.UnpackV1.Value, + FileHeader, + new MultiVolumeReadOnlyStream(Parts.Cast(), archive) + ); + } + + stream = new RarStream( + archive.UnpackV2017.Value, + FileHeader, + new MultiVolumeReadOnlyStream(Parts.Cast(), archive) + ); + await stream.InitializeAsync(cancellationToken); + return stream; } public bool IsComplete diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 5473370f4..28386c0e0 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; using SharpCompress.IO; @@ -31,7 +33,7 @@ void IStreamStack.SetPosition(long position) { } private uint currentCrc; private readonly bool disableCRC; - public RarCrcStream( + private RarCrcStream( IRarUnpack unpack, FileHeader fileHeader, MultiVolumeReadOnlyStream readStream @@ -46,6 +48,26 @@ MultiVolumeReadOnlyStream readStream ResetCrc(); } + public static RarCrcStream Create( + IRarUnpack unpack, + FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream) + { + var stream = new RarCrcStream(unpack, fileHeader, readStream); + stream.Initialize(); + return stream; + } + + public static async Task CreateAsync( + IRarUnpack unpack, + FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream, CancellationToken cancellationToken = default) + { + var stream = new RarCrcStream(unpack, fileHeader, readStream); + await stream.InitializeAsync(cancellationToken); + return stream; + } + protected override void Dispose(bool disposing) { #if DEBUG_STREAMS diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index 43d6ff4ae..2a9facaaa 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -3,6 +3,8 @@ using System; using System.Buffers; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.Rar.Headers; using SharpCompress.IO; @@ -56,13 +58,24 @@ public RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream) #if DEBUG_STREAMS this.DebugConstruct(typeof(RarStream)); #endif + } + public void Initialize() + { fetch = true; unpack.DoUnpack(fileHeader, readStream, this); fetch = false; _position = 0; } + public async Task InitializeAsync(CancellationToken cancellationToken = default) + { + fetch = true; + await unpack.DoUnpackAsync(fileHeader, readStream, this, cancellationToken); + fetch = false; + _position = 0; + } + protected override void Dispose(bool disposing) { if (!isDisposed) diff --git a/src/SharpCompress/Readers/Rar/RarReader.cs b/src/SharpCompress/Readers/Rar/RarReader.cs index 79842e6ab..8238e6197 100644 --- a/src/SharpCompress/Readers/Rar/RarReader.cs +++ b/src/SharpCompress/Readers/Rar/RarReader.cs @@ -113,7 +113,7 @@ protected override EntryStream GetEntryStream() ); if (Entry.IsRarV3) { - return CreateEntryStream(new RarCrcStream(UnpackV1.Value, Entry.FileHeader, stream)); + return CreateEntryStream( RarCrcStream.Create(UnpackV1.Value, Entry.FileHeader, stream)); } if (Entry.FileHeader.FileCrc?.Length > 5) @@ -123,6 +123,6 @@ protected override EntryStream GetEntryStream() ); } - return CreateEntryStream(new RarCrcStream(UnpackV2017.Value, Entry.FileHeader, stream)); + return CreateEntryStream(RarCrcStream.Create(UnpackV2017.Value, Entry.FileHeader, stream)); } } From 8e7d959cf4876778361c2754e85f900fb3767fdd Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 28 Oct 2025 16:07:16 +0000 Subject: [PATCH 04/25] add async creations --- .../Compressors/Rar/RarBLAKE2spStream.cs | 24 +- .../Compressors/Rar/RarStream.cs | 12 + src/SharpCompress/Readers/AbstractReader.cs | 12 +- src/SharpCompress/Readers/Rar/RarReader.cs | 42 +- .../Rar/RarReaderAsyncTests.cs | 380 ++++++++++++++++++ 5 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 tests/SharpCompress.Test/Rar/RarReaderAsyncTests.cs diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs index 5de6bf032..239d5c9da 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; using SharpCompress.IO; @@ -103,7 +105,7 @@ public BLAKE2SP() byte[] _hash = { }; - public RarBLAKE2spStream( + private RarBLAKE2spStream( IRarUnpack unpack, FileHeader fileHeader, MultiVolumeReadOnlyStream readStream @@ -121,6 +123,26 @@ MultiVolumeReadOnlyStream readStream ResetCrc(); } + public static RarBLAKE2spStream Create( + IRarUnpack unpack, + FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream) + { + var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); + stream.Initialize(); + return stream; + } + + public static async Task CreateAsync( + IRarUnpack unpack, + FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream, CancellationToken cancellationToken = default) + { + var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); + await stream.InitializeAsync(cancellationToken); + return stream; + } + protected override void Dispose(bool disposing) { #if DEBUG_STREAMS diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index 2a9facaaa..089489d46 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -237,6 +237,18 @@ public override void Write(byte[] buffer, int offset, int count) } } + public override System.Threading.Tasks.Task WriteAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + Write(buffer, offset, count); + return System.Threading.Tasks.Task.CompletedTask; + } + private void EnsureBufferCapacity(int count) { if (this.tmpBuffer.Length < this.tmpCount + count) diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index 9277753d7..17b74c2ae 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -294,7 +294,9 @@ public EntryStream OpenEntryStream() return stream; } - public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) + public async Task OpenEntryStreamAsync( + CancellationToken cancellationToken = default + ) { if (_wroteCurrentEntry) { @@ -302,9 +304,9 @@ public Task OpenEntryStreamAsync(CancellationToken cancellationToke "WriteEntryToAsync or OpenEntryStreamAsync can only be called once." ); } - var stream = GetEntryStream(); + var stream = await GetEntryStreamAsync(cancellationToken).ConfigureAwait(false); _wroteCurrentEntry = true; - return Task.FromResult(stream); + return stream; } /// @@ -316,6 +318,10 @@ protected EntryStream CreateEntryStream(Stream? decompressed) => protected virtual EntryStream GetEntryStream() => CreateEntryStream(Entry.Parts.First().GetCompressedStream()); + protected virtual Task GetEntryStreamAsync( + CancellationToken cancellationToken = default + ) => Task.FromResult(GetEntryStream()); + #endregion IEntry IReader.Entry => Entry; diff --git a/src/SharpCompress/Readers/Rar/RarReader.cs b/src/SharpCompress/Readers/Rar/RarReader.cs index 8238e6197..67af26842 100644 --- a/src/SharpCompress/Readers/Rar/RarReader.cs +++ b/src/SharpCompress/Readers/Rar/RarReader.cs @@ -113,16 +113,54 @@ protected override EntryStream GetEntryStream() ); if (Entry.IsRarV3) { - return CreateEntryStream( RarCrcStream.Create(UnpackV1.Value, Entry.FileHeader, stream)); + return CreateEntryStream(RarCrcStream.Create(UnpackV1.Value, Entry.FileHeader, stream)); } if (Entry.FileHeader.FileCrc?.Length > 5) { return CreateEntryStream( - new RarBLAKE2spStream(UnpackV2017.Value, Entry.FileHeader, stream) + RarBLAKE2spStream.Create(UnpackV2017.Value, Entry.FileHeader, stream) ); } return CreateEntryStream(RarCrcStream.Create(UnpackV2017.Value, Entry.FileHeader, stream)); } + + protected override async System.Threading.Tasks.Task GetEntryStreamAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + if (Entry.IsRedir) + { + throw new InvalidOperationException("no stream for redirect entry"); + } + + var stream = new MultiVolumeReadOnlyStream( + CreateFilePartEnumerableForCurrentEntry().Cast(), + this + ); + if (Entry.IsRarV3) + { + return CreateEntryStream( + await RarCrcStream + .CreateAsync(UnpackV1.Value, Entry.FileHeader, stream, cancellationToken) + .ConfigureAwait(false) + ); + } + + if (Entry.FileHeader.FileCrc?.Length > 5) + { + return CreateEntryStream( + await RarBLAKE2spStream + .CreateAsync(UnpackV2017.Value, Entry.FileHeader, stream, cancellationToken) + .ConfigureAwait(false) + ); + } + + return CreateEntryStream( + await RarCrcStream + .CreateAsync(UnpackV2017.Value, Entry.FileHeader, stream, cancellationToken) + .ConfigureAwait(false) + ); + } } diff --git a/tests/SharpCompress.Test/Rar/RarReaderAsyncTests.cs b/tests/SharpCompress.Test/Rar/RarReaderAsyncTests.cs new file mode 100644 index 000000000..9e07e397e --- /dev/null +++ b/tests/SharpCompress.Test/Rar/RarReaderAsyncTests.cs @@ -0,0 +1,380 @@ +using System; +using System.Collections; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives.Rar; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Readers.Rar; +using Xunit; + +namespace SharpCompress.Test.Rar; + +public class RarReaderAsyncTests : ReaderTests +{ + [Fact] + public async Task Rar_Multi_Reader_Async() => + await DoRar_Multi_Reader_Async( + [ + "Rar.multi.part01.rar", + "Rar.multi.part02.rar", + "Rar.multi.part03.rar", + "Rar.multi.part04.rar", + "Rar.multi.part05.rar", + "Rar.multi.part06.rar", + ] + ); + + [Fact] + public async Task Rar5_Multi_Reader_Async() => + await DoRar_Multi_Reader_Async( + [ + "Rar5.multi.part01.rar", + "Rar5.multi.part02.rar", + "Rar5.multi.part03.rar", + "Rar5.multi.part04.rar", + "Rar5.multi.part05.rar", + "Rar5.multi.part06.rar", + ] + ); + + private async Task DoRar_Multi_Reader_Async(string[] archives) + { + using ( + var reader = RarReader.Open( + archives + .Select(s => Path.Combine(TEST_ARCHIVES_PATH, s)) + .Select(p => File.OpenRead(p)) + ) + ) + { + while (reader.MoveToNextEntry()) + { + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_Multi_Reader_Encrypted_Async() => + await Assert.ThrowsAsync(async () => + { + string[] archives = + [ + "Rar.EncryptedParts.part01.rar", + "Rar.EncryptedParts.part02.rar", + "Rar.EncryptedParts.part03.rar", + "Rar.EncryptedParts.part04.rar", + "Rar.EncryptedParts.part05.rar", + "Rar.EncryptedParts.part06.rar", + ]; + using ( + var reader = RarReader.Open( + archives + .Select(s => Path.Combine(TEST_ARCHIVES_PATH, s)) + .Select(p => File.OpenRead(p)), + new ReaderOptions { Password = "test" } + ) + ) + { + while (reader.MoveToNextEntry()) + { + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + }); + + [Fact] + public async Task Rar_Multi_Reader_Delete_Files_Async() => + await DoRar_Multi_Reader_Delete_Files_Async( + [ + "Rar.multi.part01.rar", + "Rar.multi.part02.rar", + "Rar.multi.part03.rar", + "Rar.multi.part04.rar", + "Rar.multi.part05.rar", + "Rar.multi.part06.rar", + ] + ); + + [Fact] + public async Task Rar5_Multi_Reader_Delete_Files_Async() => + await DoRar_Multi_Reader_Delete_Files_Async( + [ + "Rar5.multi.part01.rar", + "Rar5.multi.part02.rar", + "Rar5.multi.part03.rar", + "Rar5.multi.part04.rar", + "Rar5.multi.part05.rar", + "Rar5.multi.part06.rar", + ] + ); + + private async Task DoRar_Multi_Reader_Delete_Files_Async(string[] archives) + { + foreach (var file in archives) + { + File.Copy( + Path.Combine(TEST_ARCHIVES_PATH, file), + Path.Combine(SCRATCH2_FILES_PATH, file) + ); + } + var streams = archives + .Select(s => Path.Combine(SCRATCH2_FILES_PATH, s)) + .Select(File.OpenRead) + .ToList(); + using (var reader = RarReader.Open(streams)) + { + while (reader.MoveToNextEntry()) + { + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + foreach (var stream in streams) + { + stream.Dispose(); + } + VerifyFiles(); + + foreach (var file in archives.Select(s => Path.Combine(SCRATCH2_FILES_PATH, s))) + { + File.Delete(file); + } + } + + [Fact] + public async Task Rar_None_Reader_Async() => + await ReadAsync("Rar.none.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_None_Reader_Async() => + await ReadAsync("Rar5.none.rar", CompressionType.Rar); + + [Fact] + public async Task Rar_Reader_Async() => await ReadAsync("Rar.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_Reader_Async() => await ReadAsync("Rar5.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_CRC_Blake2_Reader_Async() => + await ReadAsync("Rar5.crc_blake2.rar", CompressionType.Rar); + + [Fact] + public async Task Rar_EncryptedFileAndHeader_Reader_Async() => + await ReadRar_Async("Rar.encrypted_filesAndHeader.rar", "test"); + + [Fact] + public async Task Rar5_EncryptedFileAndHeader_Reader_Async() => + await ReadRar_Async("Rar5.encrypted_filesAndHeader.rar", "test"); + + [Fact] + public async Task Rar_EncryptedFileOnly_Reader_Async() => + await ReadRar_Async("Rar.encrypted_filesOnly.rar", "test"); + + [Fact] + public async Task Rar5_EncryptedFileOnly_Reader_Async() => + await ReadRar_Async("Rar5.encrypted_filesOnly.rar", "test"); + + [Fact] + public async Task Rar_Encrypted_Reader_Async() => + await ReadRar_Async("Rar.Encrypted.rar", "test"); + + [Fact] + public async Task Rar5_Encrypted_Reader_Async() => + await ReadRar_Async("Rar5.encrypted_filesOnly.rar", "test"); + + private async Task ReadRar_Async(string testArchive, string password) => + await ReadAsync( + testArchive, + CompressionType.Rar, + new ReaderOptions { Password = password } + ); + + [Fact] + public async Task Rar_Entry_Stream_Async() => await DoRar_Entry_Stream_Async("Rar.rar"); + + [Fact] + public async Task Rar5_Entry_Stream_Async() => await DoRar_Entry_Stream_Async("Rar5.rar"); + + private async Task DoRar_Entry_Stream_Async(string filename) + { + using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename))) + using (var reader = ReaderFactory.Open(stream)) + { + while (await reader.MoveToNextEntryAsync()) + { + if (!reader.Entry.IsDirectory) + { + Assert.Equal(CompressionType.Rar, reader.Entry.CompressionType); + var entryStream = await reader.OpenEntryStreamAsync(); + try + { + var file = Path.GetFileName(reader.Entry.Key).NotNull(); + var folder = + Path.GetDirectoryName(reader.Entry.Key) + ?? throw new ArgumentNullException(); + var destdir = Path.Combine(SCRATCH_FILES_PATH, folder); + if (!Directory.Exists(destdir)) + { + Directory.CreateDirectory(destdir); + } + var destinationFileName = Path.Combine(destdir, file); + + using var fs = File.OpenWrite(destinationFileName); + await entryStream.CopyToAsync(fs); + } + finally + { +#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + await entryStream.DisposeAsync(); +#else + entryStream.Dispose(); +#endif + } + } + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_Reader_Audio_program_Async() + { + using ( + var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Rar.Audio_program.rar")) + ) + using (var reader = ReaderFactory.Open(stream, new ReaderOptions { LookForHeader = true })) + { + while (reader.MoveToNextEntry()) + { + Assert.Equal(CompressionType.Rar, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + CompareFilesByPath( + Path.Combine(SCRATCH_FILES_PATH, "test.dat"), + Path.Combine(MISC_TEST_FILES_PATH, "test.dat") + ); + } + + [Fact] + public async Task Rar_Jpg_Reader_Async() + { + using (var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Rar.jpeg.jpg"))) + using (var reader = RarReader.Open(stream, new ReaderOptions { LookForHeader = true })) + { + while (reader.MoveToNextEntry()) + { + Assert.Equal(CompressionType.Rar, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_Solid_Reader_Async() => + await ReadAsync("Rar.solid.rar", CompressionType.Rar); + + [Fact] + public async Task Rar_Comment_Reader_Async() => + await ReadAsync("Rar.comment.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_Comment_Reader_Async() => + await ReadAsync("Rar5.comment.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_Solid_Reader_Async() => + await ReadAsync("Rar5.solid.rar", CompressionType.Rar); + + [Fact] + public async Task Rar_Solid_Skip_Reader_Async() => + await DoRar_Solid_Skip_Reader_Async("Rar.solid.rar"); + + [Fact] + public async Task Rar5_Solid_Skip_Reader_Async() => + await DoRar_Solid_Skip_Reader_Async("Rar5.solid.rar"); + + private async Task DoRar_Solid_Skip_Reader_Async(string filename) + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using var reader = ReaderFactory.Open(stream, new ReaderOptions { LookForHeader = true }); + while (reader.MoveToNextEntry()) + { + if (reader.Entry.Key.NotNull().Contains("jpg")) + { + Assert.Equal(CompressionType.Rar, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + } + + [Fact] + public async Task Rar_Reader_Skip_Async() => await DoRar_Reader_Skip_Async("Rar.rar"); + + [Fact] + public async Task Rar5_Reader_Skip_Async() => await DoRar_Reader_Skip_Async("Rar5.rar"); + + private async Task DoRar_Reader_Skip_Async(string filename) + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using var reader = ReaderFactory.Open(stream, new ReaderOptions { LookForHeader = true }); + while (reader.MoveToNextEntry()) + { + if (reader.Entry.Key.NotNull().Contains("jpg")) + { + Assert.Equal(CompressionType.Rar, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + } + + private async Task ReadAsync( + string testArchive, + CompressionType expectedCompression, + ReaderOptions? readerOptions = null + ) + { + testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive); + using Stream stream = File.OpenRead(testArchive); + using var reader = ReaderFactory.Open(stream, readerOptions ?? new ReaderOptions()); + while (await reader.MoveToNextEntryAsync()) + { + if (!reader.Entry.IsDirectory) + { + Assert.Equal(expectedCompression, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } +} From aca97c2c6cd29a6479d3588b56491a2c765d1b82 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 28 Oct 2025 16:48:05 +0000 Subject: [PATCH 05/25] add rarcrc tests --- .../Compressors/Rar/RarCRCTest.cs | 362 ++++++++++++++++++ tests/SharpCompress.Test/GZip/AsyncTests.cs | 10 +- .../Zip/ZipArchiveAsyncTests.cs | 3 +- 3 files changed, 370 insertions(+), 5 deletions(-) create mode 100644 tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs diff --git a/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs b/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs new file mode 100644 index 000000000..46206111c --- /dev/null +++ b/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs @@ -0,0 +1,362 @@ +using System; +using SharpCompress.Compressors.Rar; +using Xunit; + +namespace SharpCompress.Test.Compressors.Rar; + +public class RarCRCTest +{ + [Fact] + public void CheckCrc_SingleByte_ReturnsCorrectCrc() + { + // Arrange + uint startCrc = 0; + byte testByte = 0x42; + + // Act + var result = RarCRC.CheckCrc(startCrc, testByte); + + // Assert + Assert.NotEqual(0u, result); + } + + [Fact] + public void CheckCrc_SingleByte_WithNonZeroStartCrc() + { + // Arrange + uint startCrc = 0x12345678; + byte testByte = 0xAB; + + // Act + var result = RarCRC.CheckCrc(startCrc, testByte); + + // Assert + Assert.NotEqual(startCrc, result); + } + + [Fact] + public void CheckCrc_SingleByte_DifferentBytesProduceDifferentCrcs() + { + // Arrange + uint startCrc = 0; + byte byte1 = 0x01; + byte byte2 = 0x02; + + // Act + var result1 = RarCRC.CheckCrc(startCrc, byte1); + var result2 = RarCRC.CheckCrc(startCrc, byte2); + + // Assert + Assert.NotEqual(result1, result2); + } + + [Fact] + public void CheckCrc_EmptySpan_ReturnsStartCrc() + { + // Arrange + uint startCrc = 0x12345678; + ReadOnlySpan data = ReadOnlySpan.Empty; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 0); + + // Assert + Assert.Equal(startCrc, result); + } + + [Fact] + public void CheckCrc_SingleByteSpan_MatchesSingleByteMethod() + { + // Arrange + uint startCrc = 0; + byte testByte = 0x42; + ReadOnlySpan data = stackalloc byte[] { testByte }; + + // Act + var resultSingleByte = RarCRC.CheckCrc(startCrc, testByte); + var resultSpan = RarCRC.CheckCrc(startCrc, data, 0, 1); + + // Assert + Assert.Equal(resultSingleByte, resultSpan); + } + + [Fact] + public void CheckCrc_MultipleBytes_ProducesConsistentResult() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03, 0x04 }; + + // Act + var result1 = RarCRC.CheckCrc(startCrc, data, 0, 4); + var result2 = RarCRC.CheckCrc(startCrc, data, 0, 4); + + // Assert + Assert.Equal(result1, result2); + } + + [Fact] + public void CheckCrc_MultipleBytes_IncrementalMatchesComplete() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03, 0x04 }; + + // Act - calculate incrementally + var crc1 = RarCRC.CheckCrc(startCrc, data[0]); + var crc2 = RarCRC.CheckCrc(crc1, data[1]); + var crc3 = RarCRC.CheckCrc(crc2, data[2]); + var crc4 = RarCRC.CheckCrc(crc3, data[3]); + + // Act - calculate all at once + var crcComplete = RarCRC.CheckCrc(startCrc, data, 0, 4); + + // Assert + Assert.Equal(crc4, crcComplete); + } + + [Fact] + public void CheckCrc_WithOffset_ProcessesCorrectBytes() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[] { 0xFF, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0xFF }; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 2, 3); + var expected = RarCRC.CheckCrc(startCrc, stackalloc byte[] { 0x01, 0x02, 0x03 }, 0, 3); + + // Assert + Assert.Equal(expected, result); + } + + [Fact] + public void CheckCrc_WithCountSmallerThanData_ProcessesOnlyCount() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 3); + var expected = RarCRC.CheckCrc(startCrc, stackalloc byte[] { 0x01, 0x02, 0x03 }, 0, 3); + + // Assert + Assert.Equal(expected, result); + } + + [Fact] + public void CheckCrc_CountLargerThanRemainingData_ProcessesOnlyAvailableData() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03 }; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 100); + var expected = RarCRC.CheckCrc(startCrc, data, 0, 3); + + // Assert + Assert.Equal(expected, result); + } + + [Fact] + public void CheckCrc_KnownTestVector_HelloWorld() + { + // Arrange - "Hello, World!" in ASCII + uint startCrc = 0xFFFFFFFF; // CRC32 typically starts with inverted bits + var data = System.Text.Encoding.ASCII.GetBytes("Hello, World!"); + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, data.Length); + + // Assert - verify it produces a result (exact value depends on CRC32 variant) + Assert.NotEqual(startCrc, result); + Assert.NotEqual(0u, result); + } + + [Fact] + public void CheckCrc_AllZeros_ProducesConsistentResult() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data = stackalloc byte[10]; // all zeros + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 10); + + // Assert - verify it's deterministic + var result2 = RarCRC.CheckCrc(startCrc, data, 0, 10); + Assert.Equal(result, result2); + // CRC of all zeros from startCrc=0 can be 0, that's valid + } + + [Fact] + public void CheckCrc_AllOnes_ProducesConsistentResult() + { + // Arrange + uint startCrc = 0; + Span data = stackalloc byte[10]; + data.Fill(0xFF); + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 10); + + // Assert + var result2 = RarCRC.CheckCrc(startCrc, data, 0, 10); + Assert.Equal(result, result2); + Assert.NotEqual(0u, result); + } + + [Fact] + public void CheckCrc_OrderMatters() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan data1 = stackalloc byte[] { 0x01, 0x02 }; + ReadOnlySpan data2 = stackalloc byte[] { 0x02, 0x01 }; + + // Act + var result1 = RarCRC.CheckCrc(startCrc, data1, 0, 2); + var result2 = RarCRC.CheckCrc(startCrc, data2, 0, 2); + + // Assert - different order should produce different CRC + Assert.NotEqual(result1, result2); + } + + [Fact] + public void CheckCrc_LargeData_ProcessesCorrectly() + { + // Arrange + uint startCrc = 0; + var data = new byte[1024]; + for (int i = 0; i < data.Length; i++) + { + data[i] = (byte)(i % 256); + } + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, data.Length); + + // Assert + Assert.NotEqual(0u, result); + // Verify it's deterministic + var result2 = RarCRC.CheckCrc(startCrc, data, 0, data.Length); + Assert.Equal(result, result2); + } + + [Fact] + public void CheckCrc_PartialSpan_WithOffsetAndCount() + { + // Arrange + uint startCrc = 0; + var data = new byte[100]; + for (int i = 0; i < data.Length; i++) + { + data[i] = (byte)(i % 256); + } + + // Act - process middle section + var result = RarCRC.CheckCrc(startCrc, data, 25, 50); + + // Assert - verify it processes exactly 50 bytes starting at offset 25 + var middleSection = new byte[50]; + Array.Copy(data, 25, middleSection, 0, 50); + var expected = RarCRC.CheckCrc(startCrc, middleSection, 0, 50); + Assert.Equal(expected, result); + } + + [Fact] + public void CheckCrc_ZeroCount_ReturnsStartCrc() + { + // Arrange + uint startCrc = 0x12345678; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03 }; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 0); + + // Assert + Assert.Equal(startCrc, result); + } + + [Fact] + public void CheckCrc_MaxByteValue_HandlesCorrectly() + { + // Arrange + uint startCrc = 0; + byte maxByte = 0xFF; + + // Act + var result = RarCRC.CheckCrc(startCrc, maxByte); + + // Assert + Assert.NotEqual(0u, result); + } + + [Fact] + public void CheckCrc_MinByteValue_HandlesCorrectly() + { + // Arrange + uint startCrc = 0; + byte minByte = 0x00; + + // Act + var result = RarCRC.CheckCrc(startCrc, minByte); + + // Assert - CRC of 0x00 from startCrc=0 can be 0, that's mathematically valid + // What matters is that it's deterministic and doesn't crash + var result2 = RarCRC.CheckCrc(startCrc, minByte); + Assert.Equal(result, result2); + } + + [Fact] + public void CheckCrc_ChainedCalls_ProduceCorrectResult() + { + // Arrange + uint startCrc = 0; + ReadOnlySpan part1 = stackalloc byte[] { 0x01, 0x02 }; + ReadOnlySpan part2 = stackalloc byte[] { 0x03, 0x04 }; + ReadOnlySpan combined = stackalloc byte[] { 0x01, 0x02, 0x03, 0x04 }; + + // Act + var crc1 = RarCRC.CheckCrc(startCrc, part1, 0, 2); + var crc2 = RarCRC.CheckCrc(crc1, part2, 0, 2); + var crcCombined = RarCRC.CheckCrc(startCrc, combined, 0, 4); + + // Assert - chained calculation should equal combined calculation + Assert.Equal(crc2, crcCombined); + } + + [Theory] + [InlineData(0x00000000)] + [InlineData(0xFFFFFFFF)] + [InlineData(0x12345678)] + [InlineData(0xABCDEF01)] + public void CheckCrc_VariousStartCrcs_ProduceDifferentResults(uint startCrc) + { + // Arrange + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03 }; + + // Act + var result = RarCRC.CheckCrc(startCrc, data, 0, 3); + + // Assert - result should be different from start (unless by extreme coincidence) + Assert.NotEqual(0u, result); + } + + [Fact] + public void CheckCrc_OffsetAtEndOfData_ReturnsStartCrc() + { + // Arrange + uint startCrc = 0x12345678; + ReadOnlySpan data = stackalloc byte[] { 0x01, 0x02, 0x03 }; + + // Act - offset is at the end, so no bytes to process + var result = RarCRC.CheckCrc(startCrc, data, 3, 5); + + // Assert + Assert.Equal(startCrc, result); + } +} + diff --git a/tests/SharpCompress.Test/GZip/AsyncTests.cs b/tests/SharpCompress.Test/GZip/AsyncTests.cs index 5f2ed4fe4..2bda7a860 100644 --- a/tests/SharpCompress.Test/GZip/AsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/AsyncTests.cs @@ -6,6 +6,8 @@ using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Common; +using SharpCompress.Compressors; +using SharpCompress.Compressors.Deflate; using SharpCompress.Readers; using SharpCompress.Writers; using Xunit; @@ -192,9 +194,9 @@ public async Task CompressionStream_Async_ReadWrite() // Test async write with GZipStream using (var fileStream = File.Create(compressedPath)) using ( - var gzipStream = new Compressors.Deflate.GZipStream( + var gzipStream = new GZipStream( fileStream, - Compressors.CompressionMode.Compress + CompressionMode.Compress ) ) { @@ -208,9 +210,9 @@ public async Task CompressionStream_Async_ReadWrite() // Test async read with GZipStream using (var fileStream = File.OpenRead(compressedPath)) using ( - var gzipStream = new Compressors.Deflate.GZipStream( + var gzipStream = new GZipStream( fileStream, - Compressors.CompressionMode.Decompress + CompressionMode.Decompress ) ) { diff --git a/tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs b/tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs index 798c15b19..01fe9ca83 100644 --- a/tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs @@ -6,6 +6,7 @@ using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Common; +using SharpCompress.Compressors.Deflate; using SharpCompress.Writers; using SharpCompress.Writers.Zip; using Xunit; @@ -166,7 +167,7 @@ public async Task Zip_Create_New_Async() using (var archive = ZipArchive.Create()) { - archive.DeflateCompressionLevel = Compressors.Deflate.CompressionLevel.BestSpeed; + archive.DeflateCompressionLevel = CompressionLevel.BestSpeed; archive.AddAllFromDirectory(ORIGINAL_FILES_PATH); WriterOptions writerOptions = new ZipWriterOptions(CompressionType.Deflate); From bb53d1e1c6df31399e6362f9f51c2b8807c32ba7 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 08:41:05 +0000 Subject: [PATCH 06/25] entrystream fixes and fmt --- src/SharpCompress/Common/EntryStream.cs | 27 ++++++++----- .../Compressors/Rar/IRarUnpack.cs | 7 +++- .../Compressors/Rar/RarBLAKE2spStream.cs | 7 +++- .../Compressors/Rar/RarCrcStream.cs | 40 ++++++++++++------- .../Compressors/Rar/UnpackV1/Unpack.cs | 2 +- .../Compressors/Rar/UnpackV1/Unpack50.cs | 2 - .../Compressors/Rar/RarCRCTest.cs | 1 - tests/SharpCompress.Test/GZip/AsyncTests.cs | 14 +------ 8 files changed, 57 insertions(+), 43 deletions(-) diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index 97465e2b6..7ac2aa840 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -64,6 +64,11 @@ public async Task SkipEntryAsync(CancellationToken cancellationToken = default) protected override void Dispose(bool disposing) { + if (_isDisposed) + { + return; + } + _isDisposed = true; if (!(_completed || _reader.Cancelled)) { SkipEntry(); @@ -81,12 +86,6 @@ protected override void Dispose(bool disposing) lzmaStream.Flush(); //Lzma over reads. Knock it back } } - - if (_isDisposed) - { - return; - } - _isDisposed = true; #if DEBUG_STREAMS this.DebugDispose(typeof(EntryStream)); #endif @@ -97,6 +96,11 @@ protected override void Dispose(bool disposing) #if !NETFRAMEWORK && !NETSTANDARD2_0 public override async ValueTask DisposeAsync() { + if (_isDisposed) + { + return; + } + _isDisposed = true; if (!(_completed || _reader.Cancelled)) { await SkipEntryAsync().ConfigureAwait(false); @@ -115,10 +119,6 @@ public override async ValueTask DisposeAsync() } } - if (_isDisposed) - { - return; - } _isDisposed = true; #if DEBUG_STREAMS this.DebugDispose(typeof(EntryStream)); @@ -204,4 +204,11 @@ public override int ReadByte() public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) => throw new NotSupportedException(); } diff --git a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs index 03f3cd298..7626f4ace 100644 --- a/src/SharpCompress/Compressors/Rar/IRarUnpack.cs +++ b/src/SharpCompress/Compressors/Rar/IRarUnpack.cs @@ -10,7 +10,12 @@ internal interface IRarUnpack void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStream); void DoUnpack(); - Task DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream, CancellationToken cancellationToken); + Task DoUnpackAsync( + FileHeader fileHeader, + Stream readStream, + Stream writeStream, + CancellationToken cancellationToken + ); Task DoUnpackAsync(CancellationToken cancellationToken); // eg u/i pause/resume button diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs index 239d5c9da..693d284bd 100644 --- a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -126,7 +126,8 @@ MultiVolumeReadOnlyStream readStream public static RarBLAKE2spStream Create( IRarUnpack unpack, FileHeader fileHeader, - MultiVolumeReadOnlyStream readStream) + MultiVolumeReadOnlyStream readStream + ) { var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); stream.Initialize(); @@ -136,7 +137,9 @@ public static RarBLAKE2spStream Create( public static async Task CreateAsync( IRarUnpack unpack, FileHeader fileHeader, - MultiVolumeReadOnlyStream readStream, CancellationToken cancellationToken = default) + MultiVolumeReadOnlyStream readStream, + CancellationToken cancellationToken = default + ) { var stream = new RarBLAKE2spStream(unpack, fileHeader, readStream); await stream.InitializeAsync(cancellationToken); diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 28386c0e0..334a50b7a 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -51,7 +51,8 @@ MultiVolumeReadOnlyStream readStream public static RarCrcStream Create( IRarUnpack unpack, FileHeader fileHeader, - MultiVolumeReadOnlyStream readStream) + MultiVolumeReadOnlyStream readStream + ) { var stream = new RarCrcStream(unpack, fileHeader, readStream); stream.Initialize(); @@ -61,7 +62,9 @@ public static RarCrcStream Create( public static async Task CreateAsync( IRarUnpack unpack, FileHeader fileHeader, - MultiVolumeReadOnlyStream readStream, CancellationToken cancellationToken = default) + MultiVolumeReadOnlyStream readStream, + CancellationToken cancellationToken = default + ) { var stream = new RarCrcStream(unpack, fileHeader, readStream); await stream.InitializeAsync(cancellationToken); @@ -132,22 +135,31 @@ public override async System.Threading.Tasks.ValueTask ReadAsync( System.Threading.CancellationToken cancellationToken = default ) { - var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - if (result != 0) + cancellationToken.ThrowIfCancellationRequested(); + var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); + try { - currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span.ToArray(), 0, result); + var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (result != 0) + { + currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span, 0, result); + } + else if ( + !disableCRC + && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) + && buffer.Length != 0 + ) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + + return result; } - else if ( - !disableCRC - && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) - && buffer.Length != 0 - ) + finally { - // NOTE: we use the last FileHeader in a multipart volume to check CRC - throw new InvalidFormatException("file crc mismatch"); + System.Buffers.ArrayPool.Shared.Return(array); } - - return result; } #endif } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index c75fbb6a5..3d5504317 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -236,7 +236,7 @@ public async System.Threading.Tasks.Task DoUnpackAsync( break; case 50: // rar 5.x compression - await Unpack5Async(fileHeader.IsSolid,cancellationToken).ConfigureAwait(false); + await Unpack5Async(fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); break; default: diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs index 39765e80b..11047d6c2 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack50.cs @@ -1162,7 +1162,5 @@ private bool ReadBlockHeader() Header.TablePresent = (BlockFlags & 0x80) != 0; return true; } - - } #endif diff --git a/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs b/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs index 46206111c..6a9860d5b 100644 --- a/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs +++ b/tests/SharpCompress.Test/Compressors/Rar/RarCRCTest.cs @@ -359,4 +359,3 @@ public void CheckCrc_OffsetAtEndOfData_ReturnsStartCrc() Assert.Equal(startCrc, result); } } - diff --git a/tests/SharpCompress.Test/GZip/AsyncTests.cs b/tests/SharpCompress.Test/GZip/AsyncTests.cs index 2bda7a860..562b82e88 100644 --- a/tests/SharpCompress.Test/GZip/AsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/AsyncTests.cs @@ -193,12 +193,7 @@ public async Task CompressionStream_Async_ReadWrite() // Test async write with GZipStream using (var fileStream = File.Create(compressedPath)) - using ( - var gzipStream = new GZipStream( - fileStream, - CompressionMode.Compress - ) - ) + using (var gzipStream = new GZipStream(fileStream, CompressionMode.Compress)) { await gzipStream.WriteAsync(testData, 0, testData.Length); await gzipStream.FlushAsync(); @@ -209,12 +204,7 @@ public async Task CompressionStream_Async_ReadWrite() // Test async read with GZipStream using (var fileStream = File.OpenRead(compressedPath)) - using ( - var gzipStream = new GZipStream( - fileStream, - CompressionMode.Decompress - ) - ) + using (var gzipStream = new GZipStream(fileStream, CompressionMode.Decompress)) { var decompressed = new byte[testData.Length]; var totalRead = 0; From b354f7a3a543125017dd8c8b527357215584f7ed Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 08:47:18 +0000 Subject: [PATCH 07/25] fix logic mistake --- .../Archives/Rar/RarArchiveEntry.cs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index b795d1e68..55818b79b 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -79,12 +79,15 @@ public Stream OpenEntryStream() new MultiVolumeReadOnlyStream(Parts.Cast(), archive) ); } + else + { + stream = new RarStream( + archive.UnpackV2017.Value, + FileHeader, + new MultiVolumeReadOnlyStream(Parts.Cast(), archive) + ); + } - stream = new RarStream( - archive.UnpackV2017.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast(), archive) - ); stream.Initialize(); return stream; } @@ -100,12 +103,16 @@ public async Task OpenEntryStreamAsync(CancellationToken cancellationTok new MultiVolumeReadOnlyStream(Parts.Cast(), archive) ); } + else + { + + stream = new RarStream( + archive.UnpackV2017.Value, + FileHeader, + new MultiVolumeReadOnlyStream(Parts.Cast(), archive) + ); + } - stream = new RarStream( - archive.UnpackV2017.Value, - FileHeader, - new MultiVolumeReadOnlyStream(Parts.Cast(), archive) - ); await stream.InitializeAsync(cancellationToken); return stream; } From df2ed1e584d19e32523d723d7e7340822929c2d7 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:00:23 +0000 Subject: [PATCH 08/25] fix RarStream --- src/SharpCompress/Compressors/Rar/RarStream.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index 089489d46..6bdb31545 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -149,6 +149,14 @@ public override async System.Threading.Tasks.Task ReadAsync( int offset, int count, System.Threading.CancellationToken cancellationToken + ) => + await ReadImplAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + + private async System.Threading.Tasks.Task ReadImplAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken ) { outTotal = 0; @@ -183,7 +191,7 @@ System.Threading.CancellationToken cancellationToken } #if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER - public override System.Threading.Tasks.ValueTask ReadAsync( + public override async System.Threading.Tasks.ValueTask ReadAsync( Memory buffer, System.Threading.CancellationToken cancellationToken = default ) @@ -192,9 +200,9 @@ public override System.Threading.Tasks.ValueTask ReadAsync( var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); try { - var bytesRead = Read(array, 0, buffer.Length); + var bytesRead =await ReadImplAsync(array, 0, buffer.Length, cancellationToken).ConfigureAwait(false); new ReadOnlySpan(array, 0, bytesRead).CopyTo(buffer.Span); - return new System.Threading.Tasks.ValueTask(bytesRead); + return bytesRead; } finally { From a0c5b1cd9dfa5bb016b388aa6890ec29e154500e Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:08:08 +0000 Subject: [PATCH 09/25] add archive tests --- .../Rar/RarArchiveAsyncTests.cs | 725 ++++++++++++++++++ 1 file changed, 725 insertions(+) create mode 100644 tests/SharpCompress.Test/Rar/RarArchiveAsyncTests.cs diff --git a/tests/SharpCompress.Test/Rar/RarArchiveAsyncTests.cs b/tests/SharpCompress.Test/Rar/RarArchiveAsyncTests.cs new file mode 100644 index 000000000..1c6a33f0a --- /dev/null +++ b/tests/SharpCompress.Test/Rar/RarArchiveAsyncTests.cs @@ -0,0 +1,725 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Common; +using SharpCompress.Compressors.LZMA.Utilites; +using SharpCompress.Readers; +using Xunit; + +namespace SharpCompress.Test.Rar; + +public class RarArchiveAsyncTests : ArchiveTests +{ + [Fact] + public async Task Rar_EncryptedFileAndHeader_Archive_Async() => + await ReadRarPasswordAsync("Rar.encrypted_filesAndHeader.rar", "test"); + + [Fact] + public async Task Rar_EncryptedFileAndHeader_NoPasswordExceptionTest_Async() => + await Assert.ThrowsAsync( + typeof(CryptographicException), + async () => await ReadRarPasswordAsync("Rar.encrypted_filesAndHeader.rar", null) + ); + + [Fact] + public async Task Rar5_EncryptedFileAndHeader_Archive_Async() => + await ReadRarPasswordAsync("Rar5.encrypted_filesAndHeader.rar", "test"); + + [Fact] + public async Task Rar5_EncryptedFileAndHeader_Archive_Err_Async() => + await Assert.ThrowsAsync( + typeof(CryptographicException), + async () => await ReadRarPasswordAsync("Rar5.encrypted_filesAndHeader.rar", "failed") + ); + + [Fact] + public async Task Rar5_EncryptedFileAndHeader_NoPasswordExceptionTest_Async() => + await Assert.ThrowsAsync( + typeof(CryptographicException), + async () => await ReadRarPasswordAsync("Rar5.encrypted_filesAndHeader.rar", null) + ); + + [Fact] + public async Task Rar_EncryptedFileOnly_Archive_Async() => + await ReadRarPasswordAsync("Rar.encrypted_filesOnly.rar", "test"); + + [Fact] + public async Task Rar_EncryptedFileOnly_Archive_Err_Async() => + await Assert.ThrowsAsync( + typeof(CryptographicException), + async () => await ReadRarPasswordAsync("Rar5.encrypted_filesOnly.rar", "failed") + ); + + [Fact] + public async Task Rar5_EncryptedFileOnly_Archive_Async() => + await ReadRarPasswordAsync("Rar5.encrypted_filesOnly.rar", "test"); + + [Fact] + public async Task Rar_Encrypted_Archive_Async() => + await ReadRarPasswordAsync("Rar.Encrypted.rar", "test"); + + [Fact] + public async Task Rar5_Encrypted_Archive_Async() => + await ReadRarPasswordAsync("Rar5.encrypted_filesAndHeader.rar", "test"); + + private async Task ReadRarPasswordAsync(string testArchive, string? password) + { + using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, testArchive))) + using ( + var archive = RarArchive.Open( + stream, + new ReaderOptions { Password = password, LeaveStreamOpen = true } + ) + ) + { + foreach (var entry in archive.Entries) + { + if (!entry.IsDirectory) + { + Assert.Equal(CompressionType.Rar, entry.CompressionType); + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_Multi_Archive_Encrypted_Async() => + await Assert.ThrowsAsync(async () => + await ArchiveFileReadPasswordAsync("Rar.EncryptedParts.part01.rar", "test") + ); + + protected async Task ArchiveFileReadPasswordAsync(string archiveName, string password) + { + using ( + var archive = RarArchive.Open( + Path.Combine(TEST_ARCHIVES_PATH, archiveName), + new ReaderOptions { Password = password, LeaveStreamOpen = true } + ) + ) + { + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_None_ArchiveStreamRead_Async() => + await ArchiveStreamReadAsync("Rar.none.rar"); + + [Fact] + public async Task Rar5_None_ArchiveStreamRead_Async() => + await ArchiveStreamReadAsync("Rar5.none.rar"); + + [Fact] + public async Task Rar_ArchiveStreamRead_Async() => await ArchiveStreamReadAsync("Rar.rar"); + + [Fact] + public async Task Rar5_ArchiveStreamRead_Async() => await ArchiveStreamReadAsync("Rar5.rar"); + + [Fact] + public async Task Rar_test_invalid_exttime_ArchiveStreamRead_Async() => + await DoRar_test_invalid_exttime_ArchiveStreamReadAsync("Rar.test_invalid_exttime.rar"); + + private async Task DoRar_test_invalid_exttime_ArchiveStreamReadAsync(string filename) + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using var archive = ArchiveFactory.Open(stream); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + + [Fact] + public async Task Rar_Jpg_ArchiveStreamRead_Async() + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Rar.jpeg.jpg")); + using (var archive = RarArchive.Open(stream, new ReaderOptions { LookForHeader = true })) + { + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_IsSolidArchiveCheck_Async() => + await DoRar_IsSolidArchiveCheckAsync("Rar.rar"); + + [Fact] + public async Task Rar5_IsSolidArchiveCheck_Async() => + await DoRar_IsSolidArchiveCheckAsync("Rar5.rar"); + + private async Task DoRar_IsSolidArchiveCheckAsync(string filename) + { + using (var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename))) + { + using var archive = RarArchive.Open(stream); + Assert.False(archive.IsSolid); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_IsSolidEntryStreamCheck_Async() => + await DoRar_IsSolidEntryStreamCheckAsync("Rar.solid.rar"); + + private async Task DoRar_IsSolidEntryStreamCheckAsync(string filename) + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using var archive = RarArchive.Open(stream); + Assert.True(archive.IsSolid); + IArchiveEntry[] entries = archive.Entries.Where(a => !a.IsDirectory).ToArray(); + Assert.NotInRange(entries.Length, 0, 1); + Assert.False(entries[0].IsSolid); + var testEntry = entries[1]; + Assert.True(testEntry.IsSolid); + + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + using (var crcStream = new CrcCheckStream((uint)entry.Crc)) + { + using var eStream = await entry.OpenEntryStreamAsync(); + await eStream.CopyToAsync(crcStream); + } + if (entry == testEntry) + { + break; + } + } + } + + [Fact] + public async Task Rar_Solid_ArchiveStreamRead_Async() => + await ArchiveStreamReadAsync("Rar.solid.rar"); + + [Fact] + public async Task Rar5_Solid_ArchiveStreamRead_Async() => + await ArchiveStreamReadAsync("Rar5.solid.rar"); + + [Fact] + public async Task Rar_Solid_StreamRead_Extract_All_Async() => + await ArchiveStreamReadExtractAllAsync("Rar.solid.rar", CompressionType.Rar); + + [Fact] + public async Task Rar5_Solid_StreamRead_Extract_All_Async() => + await ArchiveStreamReadExtractAllAsync("Rar5.solid.rar", CompressionType.Rar); + + [Fact] + public async Task Rar_Multi_ArchiveStreamRead_Async() => + await DoRar_Multi_ArchiveStreamReadAsync( + [ + "Rar.multi.part01.rar", + "Rar.multi.part02.rar", + "Rar.multi.part03.rar", + "Rar.multi.part04.rar", + "Rar.multi.part05.rar", + "Rar.multi.part06.rar", + ], + false + ); + + [Fact] + public async Task Rar5_Multi_ArchiveStreamRead_Async() => + await DoRar_Multi_ArchiveStreamReadAsync( + [ + "Rar5.multi.part01.rar", + "Rar5.multi.part02.rar", + "Rar5.multi.part03.rar", + "Rar5.multi.part04.rar", + "Rar5.multi.part05.rar", + "Rar5.multi.part06.rar", + ], + false + ); + + private async Task DoRar_Multi_ArchiveStreamReadAsync(string[] archives, bool isSolid) + { + using var archive = RarArchive.Open( + archives.Select(s => Path.Combine(TEST_ARCHIVES_PATH, s)).Select(File.OpenRead) + ); + Assert.Equal(archive.IsSolid, isSolid); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + + [Fact] + public async Task Rar5_MultiSolid_ArchiveStreamRead_Async() => + await DoRar_Multi_ArchiveStreamReadAsync( + [ + "Rar.multi.solid.part01.rar", + "Rar.multi.solid.part02.rar", + "Rar.multi.solid.part03.rar", + "Rar.multi.solid.part04.rar", + "Rar.multi.solid.part05.rar", + "Rar.multi.solid.part06.rar", + ], + true + ); + + [Fact] + public async Task RarNoneArchiveFileRead_Async() => await ArchiveFileReadAsync("Rar.none.rar"); + + [Fact] + public async Task Rar5NoneArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar5.none.rar"); + + [Fact] + public async Task Rar_ArchiveFileRead_Async() => await ArchiveFileReadAsync("Rar.rar"); + + [Fact] + public async Task Rar5_ArchiveFileRead_Async() => await ArchiveFileReadAsync("Rar5.rar"); + + [Fact] + public async Task Rar_ArchiveFileRead_HasDirectories_Async() => + await DoRar_ArchiveFileRead_HasDirectoriesAsync("Rar.rar"); + + [Fact] + public async Task Rar5_ArchiveFileRead_HasDirectories_Async() => + await DoRar_ArchiveFileRead_HasDirectoriesAsync("Rar5.rar"); + + private Task DoRar_ArchiveFileRead_HasDirectoriesAsync(string filename) + { + using var stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, filename)); + using var archive = RarArchive.Open(stream); + Assert.False(archive.IsSolid); + Assert.Contains(true, archive.Entries.Select(entry => entry.IsDirectory)); + return Task.CompletedTask; + } + + [Fact] + public async Task Rar_Jpg_ArchiveFileRead_Async() + { + using ( + var archive = RarArchive.Open( + Path.Combine(TEST_ARCHIVES_PATH, "Rar.jpeg.jpg"), + new ReaderOptions { LookForHeader = true } + ) + ) + { + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + VerifyFiles(); + } + + [Fact] + public async Task Rar_Solid_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar.solid.rar"); + + [Fact] + public async Task Rar5_Solid_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar5.solid.rar"); + + [Fact] + public async Task Rar2_Multi_ArchiveStreamRead_Async() => + await DoRar_Multi_ArchiveStreamReadAsync( + [ + "Rar2.multi.rar", + "Rar2.multi.r00", + "Rar2.multi.r01", + "Rar2.multi.r02", + "Rar2.multi.r03", + "Rar2.multi.r04", + "Rar2.multi.r05", + ], + false + ); + + [Fact] + public async Task Rar2_Multi_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar2.multi.rar"); + + [Fact] + public async Task Rar2_ArchiveFileRead_Async() => await ArchiveFileReadAsync("Rar2.rar"); + + [Fact] + public async Task Rar15_ArchiveFileRead_Async() + { + UseExtensionInsteadOfNameToVerify = true; + UseCaseInsensitiveToVerify = true; + await ArchiveFileReadAsync("Rar15.rar"); + } + + [Fact] + public void Rar15_ArchiveVersionTest_Async() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Rar15.rar"); + + using var archive = RarArchive.Open(testArchive); + Assert.Equal(1, archive.MinVersion); + Assert.Equal(1, archive.MaxVersion); + } + + [Fact] + public void Rar2_ArchiveVersionTest_Async() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Rar2.rar"); + + using var archive = RarArchive.Open(testArchive); + Assert.Equal(2, archive.MinVersion); + Assert.Equal(2, archive.MaxVersion); + } + + [Fact] + public void Rar4_ArchiveVersionTest_Async() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Rar4.multi.part01.rar"); + + using var archive = RarArchive.Open(testArchive); + Assert.Equal(3, archive.MinVersion); + Assert.Equal(4, archive.MaxVersion); + } + + [Fact] + public void Rar5_ArchiveVersionTest_Async() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Rar5.solid.rar"); + + using var archive = RarArchive.Open(testArchive); + Assert.Equal(5, archive.MinVersion); + Assert.Equal(6, archive.MaxVersion); + } + + [Fact] + public async Task Rar4_Multi_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar4.multi.part01.rar"); + + [Fact] + public async Task Rar4_ArchiveFileRead_Async() => await ArchiveFileReadAsync("Rar4.rar"); + + [Fact] + public void Rar_GetPartsSplit_Async() => + ArchiveGetParts( + new[] + { + "Rar4.split.001", + "Rar4.split.002", + "Rar4.split.003", + "Rar4.split.004", + "Rar4.split.005", + "Rar4.split.006", + } + ); + + [Fact] + public void Rar_GetPartsOld_Async() => + ArchiveGetParts( + new[] + { + "Rar2.multi.rar", + "Rar2.multi.r00", + "Rar2.multi.r01", + "Rar2.multi.r02", + "Rar2.multi.r03", + "Rar2.multi.r04", + "Rar2.multi.r05", + } + ); + + [Fact] + public void Rar_GetPartsNew_Async() => + ArchiveGetParts( + new[] + { + "Rar4.multi.part01.rar", + "Rar4.multi.part02.rar", + "Rar4.multi.part03.rar", + "Rar4.multi.part04.rar", + "Rar4.multi.part05.rar", + "Rar4.multi.part06.rar", + "Rar4.multi.part07.rar", + } + ); + + [Fact] + public async Task Rar4_Multi_ArchiveStreamRead_Async() => + await DoRar_Multi_ArchiveStreamReadAsync( + [ + "Rar4.multi.part01.rar", + "Rar4.multi.part02.rar", + "Rar4.multi.part03.rar", + "Rar4.multi.part04.rar", + "Rar4.multi.part05.rar", + "Rar4.multi.part06.rar", + "Rar4.multi.part07.rar", + ], + false + ); + + [Fact] + public async Task Rar4_Split_ArchiveStreamRead_Async() => + await ArchiveStreamMultiReadAsync( + null, + [ + "Rar4.split.001", + "Rar4.split.002", + "Rar4.split.003", + "Rar4.split.004", + "Rar4.split.005", + "Rar4.split.006", + ] + ); + + [Fact] + public async Task Rar4_Multi_ArchiveFirstFileRead_Async() => + await ArchiveFileReadAsync("Rar4.multi.part01.rar"); + + [Fact] + public async Task Rar4_Split_ArchiveFirstFileRead_Async() => + await ArchiveFileReadAsync("Rar4.split.001"); + + [Fact] + public async Task Rar4_Split_ArchiveStreamFirstFileRead_Async() => + await ArchiveStreamMultiReadAsync(null, ["Rar4.split.001"]); + + [Fact] + public async Task Rar4_Split_ArchiveOpen_Async() => + await ArchiveOpenStreamReadAsync( + null, + "Rar4.split.001", + "Rar4.split.002", + "Rar4.split.003", + "Rar4.split.004", + "Rar4.split.005", + "Rar4.split.006" + ); + + [Fact] + public async Task Rar4_Multi_ArchiveOpen_Async() => + await ArchiveOpenStreamReadAsync( + null, + "Rar4.multi.part01.rar", + "Rar4.multi.part02.rar", + "Rar4.multi.part03.rar", + "Rar4.multi.part04.rar", + "Rar4.multi.part05.rar", + "Rar4.multi.part06.rar", + "Rar4.multi.part07.rar" + ); + + [Fact] + public void Rar4_Multi_ArchiveOpenEntryVolumeIndexTest_Async() => + ArchiveOpenEntryVolumeIndexTest( + [ + [0, 1], + [1, 5], + [5, 6], + ], + null, + "Rar4.multi.part01.rar", + "Rar4.multi.part02.rar", + "Rar4.multi.part03.rar", + "Rar4.multi.part04.rar", + "Rar4.multi.part05.rar", + "Rar4.multi.part06.rar", + "Rar4.multi.part07.rar" + ); + + [Fact] + public async Task Rar_Multi_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar.multi.part01.rar"); + + [Fact] + public async Task Rar5_Multi_ArchiveFileRead_Async() => + await ArchiveFileReadAsync("Rar5.multi.part01.rar"); + + [Fact] + public void Rar_IsFirstVolume_True_Async() => DoRar_IsFirstVolume_True("Rar.multi.part01.rar"); + + [Fact] + public void Rar5_IsFirstVolume_True_Async() => + DoRar_IsFirstVolume_True("Rar5.multi.part01.rar"); + + private void DoRar_IsFirstVolume_True(string firstFilename) + { + using var archive = RarArchive.Open(Path.Combine(TEST_ARCHIVES_PATH, firstFilename)); + Assert.True(archive.IsMultipartVolume()); + Assert.True(archive.IsFirstVolume()); + } + + [Fact] + public void Rar_IsFirstVolume_False_Async() => + DoRar_IsFirstVolume_False("Rar.multi.part03.rar"); + + [Fact] + public void Rar5_IsFirstVolume_False_Async() => + DoRar_IsFirstVolume_False("Rar5.multi.part03.rar"); + + private void DoRar_IsFirstVolume_False(string notFirstFilename) + { + using var archive = RarArchive.Open(Path.Combine(TEST_ARCHIVES_PATH, notFirstFilename)); + Assert.True(archive.IsMultipartVolume()); + Assert.False(archive.IsFirstVolume()); + } + + [Fact] + public async Task Rar5_CRC_Blake2_Archive_Async() => + await ArchiveFileReadAsync("Rar5.crc_blake2.rar"); + + [Fact] + void Rar_Iterate_Archive_Async() => + ArchiveFileSkip("Rar.rar", "Failure jpg exe Empty jpg\\test.jpg exe\\test.exe тест.txt"); + + [Fact] + public void Rar2_Iterate_Archive_Async() => + ArchiveFileSkip("Rar2.rar", "Failure Empty тест.txt jpg\\test.jpg exe\\test.exe jpg exe"); + + [Fact] + public void Rar4_Iterate_Archive_Async() => + ArchiveFileSkip("Rar4.rar", "Failure Empty jpg exe тест.txt jpg\\test.jpg exe\\test.exe"); + + [Fact] + public void Rar5_Iterate_Archive_Async() => + ArchiveFileSkip("Rar5.rar", "Failure jpg exe Empty тест.txt jpg\\test.jpg exe\\test.exe"); + + [Fact] + public void Rar_Encrypted_Iterate_Archive_Async() => + ArchiveFileSkip( + "Rar.encrypted_filesOnly.rar", + "Failure jpg exe Empty тест.txt jpg\\test.jpg exe\\test.exe" + ); + + [Fact] + public void Rar5_Encrypted_Iterate_Archive_Async() => + ArchiveFileSkip( + "Rar5.encrypted_filesOnly.rar", + "Failure jpg exe Empty тест.txt jpg\\test.jpg exe\\test.exe" + ); + + private async Task ArchiveStreamReadAsync(string testArchive) + { + testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive); + using var stream = File.OpenRead(testArchive); + using var archive = ArchiveFactory.Open(stream); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + VerifyFiles(); + } + + private async Task ArchiveStreamReadExtractAllAsync( + string testArchive, + CompressionType compression + ) + { + testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive); + using var stream = File.OpenRead(testArchive); + using var archive = ArchiveFactory.Open(stream); + Assert.True(archive.IsSolid); + using (var reader = archive.ExtractAllEntries()) + { + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + Assert.Equal(compression, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + } + VerifyFiles(); + + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + VerifyFiles(); + } + + private async Task ArchiveFileReadAsync(string testArchive) + { + testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive); + using var archive = ArchiveFactory.Open(testArchive); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + VerifyFiles(); + } + + private async Task ArchiveStreamMultiReadAsync( + ReaderOptions? readerOptions, + params string[] testArchives + ) + { + var paths = testArchives.Select(x => Path.Combine(TEST_ARCHIVES_PATH, x)); + using var archive = ArchiveFactory.Open(paths.Select(a => new FileInfo(a)), readerOptions); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + VerifyFiles(); + } + + private async Task ArchiveOpenStreamReadAsync( + ReaderOptions? readerOptions, + params string[] testArchives + ) + { + var paths = testArchives.Select(x => Path.Combine(TEST_ARCHIVES_PATH, x)); + using var archive = ArchiveFactory.Open(paths.Select(f => new FileInfo(f)), readerOptions); + foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory)) + { + await entry.WriteToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + VerifyFiles(); + } +} From 351e29436262dc02a350b336130907fee76b80f3 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:23:42 +0000 Subject: [PATCH 10/25] convert unpack15 and unpack20 --- .../Compressors/Rar/UnpackV1/Unpack.cs | 6 +- .../Compressors/Rar/UnpackV1/Unpack15.cs | 126 ++++++++ .../Compressors/Rar/UnpackV1/Unpack20.cs | 271 ++++++++++++++++++ 3 files changed, 399 insertions(+), 4 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index 3d5504317..ac935576d 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -217,17 +217,15 @@ public async System.Threading.Tasks.Task DoUnpackAsync( await UnstoreFileAsync(cancellationToken).ConfigureAwait(false); return; } - // TODO: When compression methods are converted to async, call them here - // For now, fall back to synchronous version switch (fileHeader.CompressionAlgorithm) { case 15: // rar 1.5 compression - unpack15(fileHeader.IsSolid); + await unpack15Async(fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); break; case 20: // rar 2.x compression case 26: // files larger than 2GB - unpack20(fileHeader.IsSolid); + await unpack20Async(fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); break; case 29: // rar 3.x compression diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs index ac962b67c..4b9db7d68 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack15.cs @@ -316,6 +316,110 @@ private void unpack15(bool solid) oldUnpWriteBuf(); } + private async System.Threading.Tasks.Task unpack15Async( + bool solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (suspended) + { + unpPtr = wrPtr; + } + else + { + UnpInitData(solid); + oldUnpInitData(solid); + await unpReadBufAsync(cancellationToken).ConfigureAwait(false); + if (!solid) + { + initHuff(); + unpPtr = 0; + } + else + { + unpPtr = wrPtr; + } + --destUnpSize; + } + if (destUnpSize >= 0) + { + getFlagsBuf(); + FlagsCnt = 8; + } + + while (destUnpSize >= 0) + { + unpPtr &= PackDef.MAXWINMASK; + + if ( + inAddr > readTop - 30 + && !await unpReadBufAsync(cancellationToken).ConfigureAwait(false) + ) + { + break; + } + if (((wrPtr - unpPtr) & PackDef.MAXWINMASK) < 270 && wrPtr != unpPtr) + { + await oldUnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + if (suspended) + { + return; + } + } + if (StMode != 0) + { + huffDecode(); + continue; + } + + if (--FlagsCnt < 0) + { + getFlagsBuf(); + FlagsCnt = 7; + } + + if ((FlagBuf & 0x80) != 0) + { + FlagBuf <<= 1; + if (Nlzb > Nhfb) + { + longLZ(); + } + else + { + huffDecode(); + } + } + else + { + FlagBuf <<= 1; + if (--FlagsCnt < 0) + { + getFlagsBuf(); + FlagsCnt = 7; + } + if ((FlagBuf & 0x80) != 0) + { + FlagBuf <<= 1; + if (Nlzb > Nhfb) + { + huffDecode(); + } + else + { + longLZ(); + } + } + else + { + FlagBuf <<= 1; + shortLZ(); + } + } + } + await oldUnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + } + private bool unpReadBuf() { var dataSize = readTop - inAddr; @@ -848,4 +952,26 @@ private void oldUnpWriteBuf() } wrPtr = unpPtr; } + + private async System.Threading.Tasks.Task oldUnpWriteBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + if (unpPtr < wrPtr) + { + await writeStream + .WriteAsync(window, wrPtr, -wrPtr & PackDef.MAXWINMASK, cancellationToken) + .ConfigureAwait(false); + await writeStream + .WriteAsync(window, 0, unpPtr, cancellationToken) + .ConfigureAwait(false); + } + else + { + await writeStream + .WriteAsync(window, wrPtr, unpPtr - wrPtr, cancellationToken) + .ConfigureAwait(false); + } + wrPtr = unpPtr; + } } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs index 69a266bb6..2b941eab2 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs @@ -368,6 +368,163 @@ private void unpack20(bool solid) oldUnpWriteBuf(); } + private async System.Threading.Tasks.Task unpack20Async( + bool solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + int Bits; + + if (suspended) + { + unpPtr = wrPtr; + } + else + { + UnpInitData(solid); + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + if (!solid) + { + if (!await ReadTables20Async(cancellationToken).ConfigureAwait(false)) + { + return; + } + } + --destUnpSize; + } + + while (destUnpSize >= 0) + { + unpPtr &= PackDef.MAXWINMASK; + + if (inAddr > readTop - 30) + { + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + break; + } + } + if (((wrPtr - unpPtr) & PackDef.MAXWINMASK) < 270 && wrPtr != unpPtr) + { + await oldUnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + if (suspended) + { + return; + } + } + if (UnpAudioBlock != 0) + { + var AudioNumber = this.decodeNumber(MD[UnpCurChannel]); + + if (AudioNumber == 256) + { + if (!await ReadTables20Async(cancellationToken).ConfigureAwait(false)) + { + break; + } + continue; + } + window[unpPtr++] = DecodeAudio(AudioNumber); + if (++UnpCurChannel == UnpChannels) + { + UnpCurChannel = 0; + } + --destUnpSize; + continue; + } + + var Number = this.decodeNumber(LD); + if (Number < 256) + { + window[unpPtr++] = (byte)Number; + --destUnpSize; + continue; + } + if (Number > 269) + { + var Length = LDecode[Number -= 270] + 3; + if ((Bits = LBits[Number]) > 0) + { + Length += Utility.URShift(GetBits(), (16 - Bits)); + AddBits(Bits); + } + + var DistNumber = this.decodeNumber(DD); + var Distance = DDecode[DistNumber] + 1; + if ((Bits = DBits[DistNumber]) > 0) + { + Distance += Utility.URShift(GetBits(), (16 - Bits)); + AddBits(Bits); + } + + if (Distance >= 0x2000) + { + Length++; + if (Distance >= 0x40000L) + { + Length++; + } + } + + CopyString20(Length, Distance); + continue; + } + if (Number == 269) + { + if (!await ReadTables20Async(cancellationToken).ConfigureAwait(false)) + { + break; + } + continue; + } + if (Number == 256) + { + CopyString20(lastLength, lastDist); + continue; + } + if (Number < 261) + { + var Distance = oldDist[(oldDistPtr - (Number - 256)) & 3]; + var LengthNumber = this.decodeNumber(RD); + var Length = LDecode[LengthNumber] + 2; + if ((Bits = LBits[LengthNumber]) > 0) + { + Length += Utility.URShift(GetBits(), (16 - Bits)); + AddBits(Bits); + } + if (Distance >= 0x101) + { + Length++; + if (Distance >= 0x2000) + { + Length++; + if (Distance >= 0x40000) + { + Length++; + } + } + } + CopyString20(Length, Distance); + continue; + } + if (Number < 270) + { + var Distance = SDDecode[Number -= 261] + 1; + if ((Bits = SDBits[Number]) > 0) + { + Distance += Utility.URShift(GetBits(), (16 - Bits)); + AddBits(Bits); + } + CopyString20(2, Distance); + } + } + ReadLastTables(); + await oldUnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + } + private void CopyString20(int Length, int Distance) { lastDist = oldDist[oldDistPtr++ & 3] = Distance; @@ -534,6 +691,120 @@ private bool ReadTables20() return (true); } + private async System.Threading.Tasks.Task ReadTables20Async( + System.Threading.CancellationToken cancellationToken = default + ) + { + byte[] BitLength = new byte[PackDef.BC20]; + byte[] Table = new byte[PackDef.MC20 * 4]; + int TableSize, + N, + I; + if (inAddr > readTop - 25) + { + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return (false); + } + } + var BitField = GetBits(); + UnpAudioBlock = (BitField & 0x8000); + + if (0 == (BitField & 0x4000)) + { + new Span(UnpOldTable20).Clear(); + } + AddBits(2); + + if (UnpAudioBlock != 0) + { + UnpChannels = ((Utility.URShift(BitField, 12)) & 3) + 1; + if (UnpCurChannel >= UnpChannels) + { + UnpCurChannel = 0; + } + AddBits(2); + TableSize = PackDef.MC20 * UnpChannels; + } + else + { + TableSize = PackDef.NC20 + PackDef.DC20 + PackDef.RC20; + } + for (I = 0; I < PackDef.BC20; I++) + { + BitLength[I] = (byte)(Utility.URShift(GetBits(), 12)); + AddBits(4); + } + UnpackUtility.makeDecodeTables(BitLength, 0, BD, PackDef.BC20); + I = 0; + while (I < TableSize) + { + if (inAddr > readTop - 5) + { + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return (false); + } + } + var Number = this.decodeNumber(BD); + if (Number < 16) + { + Table[I] = (byte)((Number + UnpOldTable20[I]) & 0xf); + I++; + } + else if (Number == 16) + { + N = (Utility.URShift(GetBits(), 14)) + 3; + AddBits(2); + while (N-- > 0 && I < TableSize) + { + Table[I] = Table[I - 1]; + I++; + } + } + else + { + if (Number == 17) + { + N = (Utility.URShift(GetBits(), 13)) + 3; + AddBits(3); + } + else + { + N = (Utility.URShift(GetBits(), 9)) + 11; + AddBits(7); + } + while (N-- > 0 && I < TableSize) + { + Table[I++] = 0; + } + } + } + if (inAddr > readTop) + { + return (true); + } + if (UnpAudioBlock != 0) + { + for (I = 0; I < UnpChannels; I++) + { + UnpackUtility.makeDecodeTables(Table, I * PackDef.MC20, MD[I], PackDef.MC20); + } + } + else + { + UnpackUtility.makeDecodeTables(Table, 0, LD, PackDef.NC20); + UnpackUtility.makeDecodeTables(Table, PackDef.NC20, DD, PackDef.DC20); + UnpackUtility.makeDecodeTables(Table, PackDef.NC20 + PackDef.DC20, RD, PackDef.RC20); + } + + for (var i = 0; i < UnpOldTable20.Length; i++) + { + UnpOldTable20[i] = Table[i]; + } + return (true); + } + private void unpInitData20(bool Solid) { if (!Solid) From aa4cd373ac33f6526008936bff2267885f1b7b62 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:28:34 +0000 Subject: [PATCH 11/25] added more async methods --- .../Compressors/Rar/UnpackV2017/Unpack.cs | 10 +++ .../Rar/UnpackV2017/Unpack.unpack30_cpp.cs | 21 ++++++ .../Rar/UnpackV2017/Unpack.unpack50_cpp.cs | 65 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index a66f0acbe..12c2c5794 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -41,6 +41,16 @@ private async System.Threading.Tasks.Task UnpIO_UnpReadAsync( private void UnpIO_UnpWrite(byte[] buf, size_t offset, uint count) => writeStream.Write(buf, checked((int)offset), checked((int)count)); + private async System.Threading.Tasks.Task UnpIO_UnpWriteAsync( + byte[] buf, + size_t offset, + uint count, + System.Threading.CancellationToken cancellationToken = default + ) => + await writeStream + .WriteAsync(buf, checked((int)offset), checked((int)count), cancellationToken) + .ConfigureAwait(false); + public void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStream) { // as of 12/2017 .NET limits array indexing to using a signed integer diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs index 7a5caee00..4848da60c 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs @@ -533,6 +533,27 @@ bool UnpReadBuf30() return ReadCode!=-1; } +async System.Threading.Tasks.Task UnpReadBuf30Async(System.Threading.CancellationToken cancellationToken = default) +{ + int DataSize=ReadTop-Inp.InAddr; // Data left to process. + if (DataSize<0) + return false; + if (Inp.InAddr>BitInput.MAX_SIZE/2) + { + if (DataSize>0) + Array.Copy(Inp.InBuf,Inp.InAddr,Inp.InBuf,0,DataSize); + Inp.InAddr=0; + ReadTop=DataSize; + } + else + DataSize=ReadTop; + int ReadCode=await UnpIO_UnpReadAsync(Inp.InBuf,DataSize,BitInput.MAX_SIZE-DataSize,cancellationToken).ConfigureAwait(false); + if (ReadCode>0) + ReadTop+=ReadCode; + ReadBorder=ReadTop-30; + return ReadCode!=-1; +} + void UnpWriteBuf30() { diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs index 106caff9c..332808489 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs @@ -664,6 +664,48 @@ private void UnpWriteArea(size_t StartPtr, size_t EndPtr) } } + private async System.Threading.Tasks.Task UnpWriteAreaAsync( + size_t StartPtr, + size_t EndPtr, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (EndPtr != StartPtr) + { + UnpSomeRead = true; + } + + if (EndPtr < StartPtr) + { + UnpAllBuf = true; + } + + if (Fragmented) + { + var SizeToWrite = (EndPtr - StartPtr) & MaxWinMask; + while (SizeToWrite > 0) + { + var BlockSize = FragWindow.GetBlockSize(StartPtr, SizeToWrite); + FragWindow.GetBuffer(StartPtr, out var __buffer, out var __offset); + await UnpWriteDataAsync(__buffer, __offset, BlockSize, cancellationToken) + .ConfigureAwait(false); + SizeToWrite -= BlockSize; + StartPtr = (StartPtr + BlockSize) & MaxWinMask; + } + } + else if (EndPtr < StartPtr) + { + await UnpWriteDataAsync(Window, StartPtr, MaxWinSize - StartPtr, cancellationToken) + .ConfigureAwait(false); + await UnpWriteDataAsync(Window, 0, EndPtr, cancellationToken).ConfigureAwait(false); + } + else + { + await UnpWriteDataAsync(Window, StartPtr, EndPtr - StartPtr, cancellationToken) + .ConfigureAwait(false); + } + } + private void UnpWriteData(byte[] Data, size_t offset, size_t Size) { if (WrittenFileSize >= DestUnpSize) @@ -682,6 +724,29 @@ private void UnpWriteData(byte[] Data, size_t offset, size_t Size) WrittenFileSize += Size; } + private async System.Threading.Tasks.Task UnpWriteDataAsync( + byte[] Data, + size_t offset, + size_t Size, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (WrittenFileSize >= DestUnpSize) + { + return; + } + + var WriteSize = Size; + var LeftToWrite = DestUnpSize - WrittenFileSize; + if (WriteSize > LeftToWrite) + { + WriteSize = (size_t)LeftToWrite; + } + + await UnpIO_UnpWriteAsync(Data, offset, WriteSize, cancellationToken).ConfigureAwait(false); + WrittenFileSize += Size; + } + private void UnpInitData50(bool Solid) { if (!Solid) From 16543bf74c4ab3bee12d1c4a3738d06951f2bd83 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:38:55 +0000 Subject: [PATCH 12/25] async UnpReadBufAsync --- .../Rar/UnpackV2017/Unpack.unpack50_cpp.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs index 332808489..1456743f6 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs @@ -339,6 +339,58 @@ private bool UnpReadBuf() return ReadCode != -1; } + private async System.Threading.Tasks.Task UnpReadBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var DataSize = ReadTop - Inp.InAddr; // Data left to process. + if (DataSize < 0) + { + return false; + } + + BlockHeader.BlockSize -= Inp.InAddr - BlockHeader.BlockStart; + if (Inp.InAddr > MAX_SIZE / 2) + { + if (DataSize > 0) + { + Buffer.BlockCopy(Inp.InBuf, Inp.InAddr, Inp.InBuf, 0, DataSize); + } + + Inp.InAddr = 0; + ReadTop = DataSize; + } + else + { + DataSize = ReadTop; + } + + var ReadCode = 0; + if (MAX_SIZE != DataSize) + { + ReadCode = await UnpIO_UnpReadAsync( + Inp.InBuf, + DataSize, + MAX_SIZE - DataSize, + cancellationToken + ) + .ConfigureAwait(false); + } + + if (ReadCode > 0) // Can be also -1. + { + ReadTop += ReadCode; + } + + ReadBorder = ReadTop - 30; + BlockHeader.BlockStart = Inp.InAddr; + if (BlockHeader.BlockSize != -1) // '-1' means not defined yet. + { + ReadBorder = Math.Min(ReadBorder, BlockHeader.BlockStart + BlockHeader.BlockSize - 1); + } + return ReadCode != -1; + } + private void UnpWriteBuf() { var WrittenBorder = WrPtr; From a09327b831a4a7b029d0c4dee8e3a97404da8e69 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:46:10 +0000 Subject: [PATCH 13/25] unpack50async --- .../Rar/UnpackV2017/Unpack.unpack50_cpp.cs | 331 ++++++++++++++++++ .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 51 +++ 2 files changed, 382 insertions(+) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs index 1456743f6..9631f1e1d 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack50_cpp.cs @@ -222,6 +222,180 @@ private void Unpack5(bool Solid) UnpWriteBuf(); } + private async System.Threading.Tasks.Task Unpack5Async( + bool Solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + FileExtracted = true; + + if (!Suspended) + { + UnpInitData(Solid); + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + if ( + !ReadBlockHeader(Inp, ref BlockHeader) + || !ReadTables(Inp, ref BlockHeader, ref BlockTables) + || !TablesRead5 + ) + { + return; + } + } + + while (true) + { + UnpPtr &= MaxWinMask; + + if (Inp.InAddr >= ReadBorder) + { + var FileDone = false; + + while ( + Inp.InAddr > BlockHeader.BlockStart + BlockHeader.BlockSize - 1 + || Inp.InAddr == BlockHeader.BlockStart + BlockHeader.BlockSize - 1 + && Inp.InBit >= BlockHeader.BlockBitSize + ) + { + if (BlockHeader.LastBlockInFile) + { + FileDone = true; + break; + } + if ( + !ReadBlockHeader(Inp, ref BlockHeader) + || !ReadTables(Inp, ref BlockHeader, ref BlockTables) + ) + { + return; + } + } + if (FileDone || !await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + break; + } + } + + if (((WriteBorder - UnpPtr) & MaxWinMask) < MAX_LZ_MATCH + 3 && WriteBorder != UnpPtr) + { + await UnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + if (WrittenFileSize > DestUnpSize) + { + return; + } + } + + uint MainSlot = DecodeNumber(Inp, BlockTables.LD); + if (MainSlot < 256) + { + if (Fragmented) + { + FragWindow[UnpPtr++] = (byte)MainSlot; + } + else + { + Window[UnpPtr++] = (byte)MainSlot; + } + continue; + } + if (MainSlot >= 262) + { + uint Length = SlotToLength(Inp, MainSlot - 262); + + uint DBits, + Distance = 1, + DistSlot = DecodeNumber(Inp, BlockTables.DD); + if (DistSlot < 4) + { + DBits = 0; + Distance += DistSlot; + } + else + { + DBits = (DistSlot / 2) - 1; + Distance += (2 | (DistSlot & 1)) << (int)DBits; + } + + if (DBits > 0) + { + if (DBits >= 4) + { + if (DBits > 4) + { + Distance += ((Inp.getbits() >> (int)(20 - DBits)) << 4); + Inp.addbits(DBits - 4); + } + + uint LowDist = DecodeNumber(Inp, BlockTables.LDD); + Distance += LowDist; + } + else + { + Distance += Inp.getbits() >> (int)(16 - DBits); + Inp.addbits(DBits); + } + } + + if (Distance > 0x100) + { + Length++; + if (Distance > 0x2000) + { + Length++; + if (Distance > 0x40000) + { + Length++; + } + } + } + + InsertOldDist(Distance); + LastLength = Length; + CopyString(Length, Distance); + continue; + } + if (MainSlot == 256) + { + var Filter = new UnpackFilter(); + if (!ReadFilter(Inp, Filter) || !AddFilter(Filter)) + { + break; + } + continue; + } + if (MainSlot == 257) + { + if (LastLength != 0) + { + CopyString((uint)LastLength, (uint)OldDist[0]); + } + continue; + } + if (MainSlot < 262) + { + uint DistNum = MainSlot - 258; + uint Distance = (uint)OldDist[(int)DistNum]; + for (var I = (int)DistNum; I > 0; I--) + { + OldDist[I] = OldDist[I - 1]; + } + OldDist[0] = Distance; + + uint LengthSlot = DecodeNumber(Inp, BlockTables.RD); + uint Length = SlotToLength(Inp, LengthSlot); + LastLength = Length; + CopyString(Length, Distance); + + continue; + } + } + await UnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + } + private uint ReadFilterData(BitInput Inp) { var ByteCount = (Inp.fgetbits() >> 14) + 1; @@ -585,6 +759,163 @@ private void UnpWriteBuf() } } + private async System.Threading.Tasks.Task UnpWriteBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + var WrittenBorder = WrPtr; + var FullWriteSize = (UnpPtr - WrittenBorder) & MaxWinMask; + var WriteSizeLeft = FullWriteSize; + var NotAllFiltersProcessed = false; + + for (var I = 0; I < Filters.Count; I++) + { + var flt = Filters[I]; + if (flt.Type == FILTER_NONE) + { + continue; + } + + if (flt.NextWindow) + { + if (((flt.BlockStart - WrPtr) & MaxWinMask) <= FullWriteSize) + { + flt.NextWindow = false; + } + continue; + } + + var BlockStart = flt.BlockStart; + var BlockLength = flt.BlockLength; + if (((BlockStart - WrittenBorder) & MaxWinMask) < WriteSizeLeft) + { + if (WrittenBorder != BlockStart) + { + await UnpWriteAreaAsync(WrittenBorder, BlockStart, cancellationToken) + .ConfigureAwait(false); + WrittenBorder = BlockStart; + WriteSizeLeft = (UnpPtr - WrittenBorder) & MaxWinMask; + } + if (BlockLength <= WriteSizeLeft) + { + if (BlockLength > 0) + { + var BlockEnd = (BlockStart + BlockLength) & MaxWinMask; + + FilterSrcMemory = EnsureCapacity( + FilterSrcMemory, + checked((int)BlockLength) + ); + var Mem = FilterSrcMemory; + if (BlockStart < BlockEnd || BlockEnd == 0) + { + if (Fragmented) + { + FragWindow.CopyData(Mem, 0, BlockStart, BlockLength); + } + else + { + Buffer.BlockCopy(Window, (int)BlockStart, Mem, 0, (int)BlockLength); + } + } + else + { + var FirstPartLength = MaxWinSize - BlockStart; + if (Fragmented) + { + FragWindow.CopyData(Mem, 0, BlockStart, FirstPartLength); + FragWindow.CopyData(Mem, FirstPartLength, 0, BlockEnd); + } + else + { + Buffer.BlockCopy( + Window, + (int)BlockStart, + Mem, + 0, + (int)FirstPartLength + ); + Buffer.BlockCopy( + Window, + 0, + Mem, + (int)FirstPartLength, + (int)BlockEnd + ); + } + } + + var OutMem = ApplyFilter(Mem, BlockLength, flt); + + Filters[I].Type = FILTER_NONE; + + if (OutMem != null) + { + await UnpIO_UnpWriteAsync(OutMem, 0, BlockLength, cancellationToken) + .ConfigureAwait(false); + WrittenFileSize += BlockLength; + } + + WrittenBorder = BlockEnd; + WriteSizeLeft = (UnpPtr - WrittenBorder) & MaxWinMask; + } + } + else + { + NotAllFiltersProcessed = true; + for (var J = I; J < Filters.Count; J++) + { + var fltj = Filters[J]; + if ( + fltj.Type != FILTER_NONE + && fltj.NextWindow == false + && ((fltj.BlockStart - WrPtr) & MaxWinMask) < FullWriteSize + ) + { + fltj.NextWindow = true; + } + } + break; + } + } + } + + var EmptyCount = 0; + for (var I = 0; I < Filters.Count; I++) + { + if (EmptyCount > 0) + { + Filters[I - EmptyCount] = Filters[I]; + } + + if (Filters[I].Type == FILTER_NONE) + { + EmptyCount++; + } + } + if (EmptyCount > 0) + { + Filters.RemoveRange(Filters.Count - EmptyCount, EmptyCount); + } + + if (!NotAllFiltersProcessed) + { + await UnpWriteAreaAsync(WrittenBorder, UnpPtr, cancellationToken).ConfigureAwait(false); + WrPtr = UnpPtr; + } + + WriteBorder = (UnpPtr + Math.Min(MaxWinSize, UNPACK_MAX_WRITE)) & MaxWinMask; + + if ( + WriteBorder == UnpPtr + || WrPtr != UnpPtr + && ((WrPtr - UnpPtr) & MaxWinMask) < ((WriteBorder - UnpPtr) & MaxWinMask) + ) + { + WriteBorder = WrPtr; + } + } + private byte[] ApplyFilter(byte[] __d, uint DataSize, UnpackFilter Flt) { var Data = 0; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index 92a37c8b0..d6ee07ecf 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -221,6 +221,57 @@ private void DoUnpack(uint Method, bool Solid) } } + private async System.Threading.Tasks.Task DoUnpackAsync( + uint Method, + bool Solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + // Methods <50 will crash in Fragmented mode when accessing NULL Window. + // They cannot be called in such mode now, but we check it below anyway + // just for extra safety. + switch (Method) + { +#if !RarV2017_SFX_MODULE + case 15: // rar 1.5 compression + if (!Fragmented) + { + // TODO: Create async version when UnpackV2017 unpack15 is converted + Unpack15(Solid); + } + + break; + case 20: // rar 2.x compression + case 26: // files larger than 2GB + if (!Fragmented) + { + // TODO: Create async version when UnpackV2017 unpack20 is converted + Unpack20(Solid); + } + + break; +#endif +#if !RarV2017_RAR5ONLY + case 29: // rar 3.x compression + if (!Fragmented) + { + // TODO: Create Unpack29Async when ready + throw new NotImplementedException(); + } + + break; +#endif + case 50: // RAR 5.0 compression algorithm. + // RAR 5.0 has full async support via UnpReadBufAsync and UnpWriteBuf + await Unpack5Async(Solid, cancellationToken).ConfigureAwait(false); + break; +#if !Rar2017_NOSTRICT + default: + throw new InvalidFormatException("unknown compression method " + Method); +#endif + } + } + private void UnpInitData(bool Solid) { if (!Solid) From 1af51aaabaf00532770963c31bfda2b31d156639 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:50:12 +0000 Subject: [PATCH 14/25] async unpack1.5 --- .../Rar/UnpackV2017/Unpack.unpack15_cpp.cs | 96 +++++++++++++++++++ .../Rar/UnpackV2017/Unpack.unpack20_cpp.cs | 30 ++++++ .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 3 +- 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack15_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack15_cpp.cs index c1886df84..9a2882b77 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack15_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack15_cpp.cs @@ -200,6 +200,102 @@ private void Unpack15(bool Solid) UnpWriteBuf20(); } + private async System.Threading.Tasks.Task Unpack15Async( + bool Solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + UnpInitData(Solid); + UnpInitData15(Solid); + await UnpReadBufAsync(cancellationToken).ConfigureAwait(false); + if (!Solid) + { + InitHuff(); + UnpPtr = 0; + } + else + { + UnpPtr = WrPtr; + } + + --DestUnpSize; + if (DestUnpSize >= 0) + { + GetFlagsBuf(); + FlagsCnt = 8; + } + + while (DestUnpSize >= 0) + { + UnpPtr &= MaxWinMask; + + if ( + Inp.InAddr > ReadTop - 30 + && !await UnpReadBufAsync(cancellationToken).ConfigureAwait(false) + ) + { + break; + } + + if (((WrPtr - UnpPtr) & MaxWinMask) < 270 && WrPtr != UnpPtr) + { + await UnpWriteBuf20Async(cancellationToken).ConfigureAwait(false); + } + + if (StMode != 0) + { + HuffDecode(); + continue; + } + + if (--FlagsCnt < 0) + { + GetFlagsBuf(); + FlagsCnt = 7; + } + + if ((FlagBuf & 0x80) != 0) + { + FlagBuf <<= 1; + if (Nlzb > Nhfb) + { + LongLZ(); + } + else + { + HuffDecode(); + } + } + else + { + FlagBuf <<= 1; + if (--FlagsCnt < 0) + { + GetFlagsBuf(); + FlagsCnt = 7; + } + if ((FlagBuf & 0x80) != 0) + { + FlagBuf <<= 1; + if (Nlzb > Nhfb) + { + HuffDecode(); + } + else + { + LongLZ(); + } + } + else + { + FlagBuf <<= 1; + ShortLZ(); + } + } + } + await UnpWriteBuf20Async(cancellationToken).ConfigureAwait(false); + } + //#define GetShortLen1(pos) ((pos)==1 ? Buf60+3:ShortLen1[pos]) private uint GetShortLen1(uint pos) => ((pos) == 1 ? (uint)(Buf60 + 3) : ShortLen1[pos]); diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs index c5970d518..7bbe595c6 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs @@ -370,6 +370,36 @@ private void UnpWriteBuf20() WrPtr = UnpPtr; } + private async System.Threading.Tasks.Task UnpWriteBuf20Async( + System.Threading.CancellationToken cancellationToken = default + ) + { + if (UnpPtr != WrPtr) + { + UnpSomeRead = true; + } + + if (UnpPtr < WrPtr) + { + await UnpIO_UnpWriteAsync( + Window, + WrPtr, + (uint)(-(int)WrPtr & MaxWinMask), + cancellationToken + ) + .ConfigureAwait(false); + await UnpIO_UnpWriteAsync(Window, 0, UnpPtr, cancellationToken).ConfigureAwait(false); + UnpAllBuf = true; + } + else + { + await UnpIO_UnpWriteAsync(Window, WrPtr, UnpPtr - WrPtr, cancellationToken) + .ConfigureAwait(false); + } + + WrPtr = UnpPtr; + } + private bool ReadTables20() { Span BitLength = stackalloc byte[checked((int)BC20)]; diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index d6ee07ecf..9433d098f 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -236,8 +236,7 @@ private async System.Threading.Tasks.Task DoUnpackAsync( case 15: // rar 1.5 compression if (!Fragmented) { - // TODO: Create async version when UnpackV2017 unpack15 is converted - Unpack15(Solid); + await Unpack15Async(Solid, cancellationToken).ConfigureAwait(false); } break; From 58bab0d310143a16d5a9422e658fd24b4a6b0363 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:53:14 +0000 Subject: [PATCH 15/25] async unpack2.0 --- .../Rar/UnpackV2017/Unpack.unpack20_cpp.cs | 288 ++++++++++++++++++ .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 3 +- 2 files changed, 289 insertions(+), 2 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs index 7bbe595c6..b95d6df05 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack20_cpp.cs @@ -349,6 +349,170 @@ private void Unpack20(bool Solid) UnpWriteBuf20(); } + private async System.Threading.Tasks.Task Unpack20Async( + bool Solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + uint Bits; + + if (Suspended) + { + UnpPtr = WrPtr; + } + else + { + UnpInitData(Solid); + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + + if ( + (!Solid || !TablesRead2) + && !await ReadTables20Async(cancellationToken).ConfigureAwait(false) + ) + { + return; + } + + --DestUnpSize; + } + + while (DestUnpSize >= 0) + { + UnpPtr &= MaxWinMask; + + if (Inp.InAddr > ReadTop - 30) + { + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + break; + } + } + + if (((WrPtr - UnpPtr) & MaxWinMask) < 270 && WrPtr != UnpPtr) + { + await UnpWriteBuf20Async(cancellationToken).ConfigureAwait(false); + if (Suspended) + { + return; + } + } + if (UnpAudioBlock) + { + var AudioNumber = DecodeNumber(Inp, MD[UnpCurChannel]); + + if (AudioNumber == 256) + { + if (!await ReadTables20Async(cancellationToken).ConfigureAwait(false)) + { + break; + } + + continue; + } + Window[UnpPtr++] = DecodeAudio((int)AudioNumber); + if (++UnpCurChannel == UnpChannels) + { + UnpCurChannel = 0; + } + + --DestUnpSize; + continue; + } + + var Number = DecodeNumber(Inp, BlockTables.LD); + if (Number < 256) + { + Window[UnpPtr++] = (byte)Number; + --DestUnpSize; + continue; + } + if (Number > 269) + { + var Length = (uint)(LDecode[Number -= 270] + 3); + if ((Bits = LBits[Number]) > 0) + { + Length += Inp.getbits() >> (int)(16 - Bits); + Inp.addbits(Bits); + } + + var DistNumber = DecodeNumber(Inp, BlockTables.DD); + var Distance = DDecode[DistNumber] + 1; + if ((Bits = DBits[DistNumber]) > 0) + { + Distance += Inp.getbits() >> (int)(16 - Bits); + Inp.addbits(Bits); + } + + if (Distance >= 0x2000) + { + Length++; + if (Distance >= 0x40000L) + { + Length++; + } + } + + CopyString20(Length, Distance); + continue; + } + if (Number == 269) + { + if (!await ReadTables20Async(cancellationToken).ConfigureAwait(false)) + { + break; + } + + continue; + } + if (Number == 256) + { + CopyString20(LastLength, LastDist); + continue; + } + if (Number < 261) + { + var Distance = OldDist[(OldDistPtr - (Number - 256)) & 3]; + var LengthNumber = DecodeNumber(Inp, BlockTables.RD); + var Length = (uint)(LDecode[LengthNumber] + 2); + if ((Bits = LBits[LengthNumber]) > 0) + { + Length += Inp.getbits() >> (int)(16 - Bits); + Inp.addbits(Bits); + } + if (Distance >= 0x101) + { + Length++; + if (Distance >= 0x2000) + { + Length++; + if (Distance >= 0x40000) + { + Length++; + } + } + } + CopyString20(Length, Distance); + continue; + } + if (Number < 270) + { + var Distance = (uint)(SDDecode[Number -= 261] + 1); + if ((Bits = SDBits[Number]) > 0) + { + Distance += Inp.getbits() >> (int)(16 - Bits); + Inp.addbits(Bits); + } + CopyString20(2, Distance); + continue; + } + } + ReadLastTables(); + await UnpWriteBuf20Async(cancellationToken).ConfigureAwait(false); + } + private void UnpWriteBuf20() { if (UnpPtr != WrPtr) @@ -520,6 +684,130 @@ private bool ReadTables20() return true; } + private async System.Threading.Tasks.Task ReadTables20Async( + System.Threading.CancellationToken cancellationToken = default + ) + { + byte[] BitLength = new byte[checked((int)BC20)]; + byte[] Table = new byte[checked((int)MC20 * 4)]; + if (Inp.InAddr > ReadTop - 25) + { + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return false; + } + } + + var BitField = Inp.getbits(); + UnpAudioBlock = (BitField & 0x8000) != 0; + + if ((BitField & 0x4000) != 0) + { + Array.Clear(UnpOldTable20, 0, UnpOldTable20.Length); + } + + Inp.addbits(2); + + uint TableSize; + if (UnpAudioBlock) + { + UnpChannels = ((BitField >> 12) & 3) + 1; + if (UnpCurChannel >= UnpChannels) + { + UnpCurChannel = 0; + } + + Inp.addbits(2); + TableSize = MC20 * UnpChannels; + } + else + { + TableSize = NC20 + DC20 + RC20; + } + + for (int I = 0; I < checked((int)BC20); I++) + { + BitLength[I] = (byte)(Inp.getbits() >> 12); + Inp.addbits(4); + } + MakeDecodeTables(BitLength, 0, BlockTables.BD, BC20); + for (int I = 0; I < checked((int)TableSize); ) + { + if (Inp.InAddr > ReadTop - 5) + { + if (!await UnpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return false; + } + } + + var Number = DecodeNumber(Inp, BlockTables.BD); + if (Number < 16) + { + Table[I] = (byte)((Number + UnpOldTable20[I]) & 0xF); + I++; + } + else if (Number < 18) + { + uint N; + if (Number == 16) + { + N = (Inp.getbits() >> 14) + 3; + Inp.addbits(2); + } + else + { + N = (Inp.getbits() >> 13) + 11; + Inp.addbits(3); + } + if (I == 0) + { + return false; + } + + while (N-- > 0 && I < checked((int)TableSize)) + { + Table[I] = Table[I - 1]; + I++; + } + } + else + { + uint N; + if (Number == 18) + { + N = (Inp.getbits() >> 13) + 3; + Inp.addbits(3); + } + else + { + N = (Inp.getbits() >> 9) + 11; + Inp.addbits(7); + } + + while (N-- > 0 && I < checked((int)TableSize)) + { + Table[I++] = 0; + } + } + } + if (UnpAudioBlock) + { + for (int I = 0; I < UnpChannels; I++) + { + MakeDecodeTables(Table, (int)(I * MC20), MD[I], MC20); + } + } + else + { + MakeDecodeTables(Table, 0, BlockTables.LD, NC20); + MakeDecodeTables(Table, (int)NC20, BlockTables.DD, DC20); + MakeDecodeTables(Table, (int)(NC20 + DC20), BlockTables.RD, RC20); + } + Array.Copy(Table, 0, this.UnpOldTable20, 0, UnpOldTable20.Length); + return true; + } + private void ReadLastTables() { if (ReadTop >= Inp.InAddr + 5) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index 9433d098f..6ee5463a5 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -244,8 +244,7 @@ private async System.Threading.Tasks.Task DoUnpackAsync( case 26: // files larger than 2GB if (!Fragmented) { - // TODO: Create async version when UnpackV2017 unpack20 is converted - Unpack20(Solid); + await Unpack20Async(Solid, cancellationToken).ConfigureAwait(false); } break; From b83e6ee4ce04080b45c237851827142f34e33fac Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 09:55:11 +0000 Subject: [PATCH 16/25] fix async usage --- src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index 12c2c5794..f466d8537 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -115,7 +115,7 @@ public async System.Threading.Tasks.Task DoUnpackAsync( { // TODO: When compression methods are converted to async, call them here // For now, fall back to synchronous version - DoUnpack(fileHeader.CompressionAlgorithm, fileHeader.IsSolid); + await DoUnpackAsync(fileHeader.CompressionAlgorithm, fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); } } From 8324114e84707cdd4c596eec4bc0900c66a22ef8 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 10:11:34 +0000 Subject: [PATCH 17/25] remove unused code --- .../Rar/UnpackV2017/Unpack.unpack30_cpp.cs | 814 ------------------ 1 file changed, 814 deletions(-) delete mode 100644 src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs deleted file mode 100644 index 4848da60c..000000000 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack30_cpp.cs +++ /dev/null @@ -1,814 +0,0 @@ -#if !Rar2017_64bit -#else -using nint = System.Int64; -using nuint = System.UInt64; -using size_t = System.UInt64; -#endif - -//using static SharpCompress.Compressors.Rar.UnpackV2017.Unpack.Unpack30Local; -/* -namespace SharpCompress.Compressors.Rar.UnpackV2017 -{ - internal partial class Unpack - { - -#if !RarV2017_RAR5ONLY -// We use it instead of direct PPM.DecodeChar call to be sure that -// we reset PPM structures in case of corrupt data. It is important, -// because these structures can be invalid after PPM.DecodeChar returned -1. -int SafePPMDecodeChar() -{ - int Ch=PPM.DecodeChar(); - if (Ch==-1) // Corrupt PPM data found. - { - PPM.CleanUp(); // Reset possibly corrupt PPM data structures. - UnpBlockType=BLOCK_LZ; // Set faster and more fail proof LZ mode. - } - return(Ch); -} - -internal static class Unpack30Local { - public static readonly byte[] LDecode={0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224}; - public static readonly byte[] LBits= {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; - public static readonly int[] DDecode = new int[DC]; - public static readonly byte[] DBits = new byte[DC]; - public static readonly int[] DBitLengthCounts= {4,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,14,0,12}; - public static readonly byte[] SDDecode={0,4,8,16,32,64,128,192}; - public static readonly byte[] SDBits= {2,2,3, 4, 5, 6, 6, 6}; -} -void Unpack29(bool Solid) -{ - uint Bits; - - if (DDecode[1]==0) - { - int Dist=0,BitLength=0,Slot=0; - for (int I=0;IReadBorder) - { - if (!UnpReadBuf30()) - break; - } - if (((WrPtr-UnpPtr) & MaxWinMask)<260 && WrPtr!=UnpPtr) - { - UnpWriteBuf30(); - if (WrittenFileSize>DestUnpSize) - return; - if (Suspended) - { - FileExtracted=false; - return; - } - } - if (UnpBlockType==BLOCK_PPM) - { - // Here speed is critical, so we do not use SafePPMDecodeChar, - // because sometimes even the inline function can introduce - // some additional penalty. - int Ch=PPM.DecodeChar(); - if (Ch==-1) // Corrupt PPM data found. - { - PPM.CleanUp(); // Reset possibly corrupt PPM data structures. - UnpBlockType=BLOCK_LZ; // Set faster and more fail proof LZ mode. - break; - } - if (Ch==PPMEscChar) - { - int NextCh=SafePPMDecodeChar(); - if (NextCh==0) // End of PPM encoding. - { - if (!ReadTables30()) - break; - continue; - } - if (NextCh==-1) // Corrupt PPM data found. - break; - if (NextCh==2) // End of file in PPM mode. - break; - if (NextCh==3) // Read VM code. - { - if (!ReadVMCodePPM()) - break; - continue; - } - if (NextCh==4) // LZ inside of PPM. - { - uint Distance=0,Length; - bool Failed=false; - for (int I=0;I<4 && !Failed;I++) - { - int _Ch=SafePPMDecodeChar(); - if (_Ch==-1) - Failed=true; - else - if (I==3) - Length=(byte)_Ch; - else - Distance=(Distance<<8)+(byte)_Ch; - } - if (Failed) - break; - - CopyString(Length+32,Distance+2); - continue; - } - if (NextCh==5) // One byte distance match (RLE) inside of PPM. - { - int Length=SafePPMDecodeChar(); - if (Length==-1) - break; - CopyString((uint)(Length+4),1); - continue; - } - // If we are here, NextCh must be 1, what means that current byte - // is equal to our 'escape' byte, so we just store it to Window. - } - Window[UnpPtr++]=(byte)Ch; - continue; - } - - uint Number=DecodeNumber(Inp,BlockTables.LD); - if (Number<256) - { - Window[UnpPtr++]=(byte)Number; - continue; - } - if (Number>=271) - { - uint Length=(uint)(LDecode[Number-=271]+3); - if ((Bits=LBits[Number])>0) - { - Length+=Inp.getbits()>>(int)(16-Bits); - Inp.addbits(Bits); - } - - uint DistNumber=DecodeNumber(Inp,BlockTables.DD); - uint Distance=(uint)(DDecode[DistNumber]+1); - if ((Bits=DBits[DistNumber])>0) - { - if (DistNumber>9) - { - if (Bits>4) - { - Distance+=((Inp.getbits()>>(int)(20-Bits))<<4); - Inp.addbits(Bits-4); - } - if (LowDistRepCount>0) - { - LowDistRepCount--; - Distance+=(uint)PrevLowDist; - } - else - { - uint LowDist=DecodeNumber(Inp,BlockTables.LDD); - if (LowDist==16) - { - LowDistRepCount=(int)(LOW_DIST_REP_COUNT-1); - Distance+=(uint)PrevLowDist; - } - else - { - Distance+=LowDist; - PrevLowDist=(int)LowDist; - } - } - } - else - { - Distance+=Inp.getbits()>>(int)(16-Bits); - Inp.addbits(Bits); - } - } - - if (Distance>=0x2000) - { - Length++; - if (Distance>=0x40000) - Length++; - } - - InsertOldDist(Distance); - LastLength=Length; - CopyString(Length,Distance); - continue; - } - if (Number==256) - { - if (!ReadEndOfBlock()) - break; - continue; - } - if (Number==257) - { - if (!ReadVMCode()) - break; - continue; - } - if (Number==258) - { - if (LastLength!=0) - CopyString(LastLength,OldDist[0]); - continue; - } - if (Number<263) - { - uint DistNum=Number-259; - uint Distance=OldDist[DistNum]; - for (uint I=DistNum;I>0;I--) - OldDist[I]=OldDist[I-1]; - OldDist[0]=Distance; - - uint LengthNumber=DecodeNumber(Inp,BlockTables.RD); - int Length=LDecode[LengthNumber]+2; - if ((Bits=LBits[LengthNumber])>0) - { - Length+=(int)(Inp.getbits()>>(int)(16-Bits)); - Inp.addbits(Bits); - } - LastLength=(uint)Length; - CopyString((uint)Length,Distance); - continue; - } - if (Number<272) - { - uint Distance=(uint)(SDDecode[Number-=263]+1); - if ((Bits=SDBits[Number])>0) - { - Distance+=Inp.getbits()>>(int)(16-Bits); - Inp.addbits(Bits); - } - InsertOldDist(Distance); - LastLength=2; - CopyString(2,Distance); - continue; - } - } - UnpWriteBuf30(); -} - - -// Return 'false' to quit unpacking the current file or 'true' to continue. -bool ReadEndOfBlock() -{ - uint BitField=Inp.getbits(); - bool NewTable,NewFile=false; - - // "1" - no new file, new table just here. - // "00" - new file, no new table. - // "01" - new file, new table (in beginning of next file). - - if ((BitField & 0x8000)!=0) - { - NewTable=true; - Inp.addbits(1); - } - else - { - NewFile=true; - NewTable=(BitField & 0x4000)!=0; - Inp.addbits(2); - } - TablesRead3=!NewTable; - - // Quit immediately if "new file" flag is set. If "new table" flag - // is present, we'll read the table in beginning of next file - // based on 'TablesRead3' 'false' value. - if (NewFile) - return false; - return ReadTables30(); // Quit only if we failed to read tables. -} - - -bool ReadVMCode() -{ - // Entire VM code is guaranteed to fully present in block defined - // by current Huffman table. Compressor checks that VM code does not cross - // Huffman block boundaries. - uint FirstByte=Inp.getbits()>>8; - Inp.addbits(8); - uint Length=(FirstByte & 7)+1; - if (Length==7) - { - Length=(Inp.getbits()>>8)+7; - Inp.addbits(8); - } - else - if (Length==8) - { - Length=Inp.getbits(); - Inp.addbits(16); - } - if (Length==0) - return false; - Array VMCode(Length); - for (uint I=0;I=ReadTop-1 && !UnpReadBuf30() && I>8; - Inp.addbits(8); - } - return AddVMCode(FirstByte,&VMCode[0],Length); -} - - -bool ReadVMCodePPM() -{ - uint FirstByte=(uint)SafePPMDecodeChar(); - if ((int)FirstByte==-1) - return false; - uint Length=(FirstByte & 7)+1; - if (Length==7) - { - int B1=SafePPMDecodeChar(); - if (B1==-1) - return false; - Length=B1+7; - } - else - if (Length==8) - { - int B1=SafePPMDecodeChar(); - if (B1==-1) - return false; - int B2=SafePPMDecodeChar(); - if (B2==-1) - return false; - Length=B1*256+B2; - } - if (Length==0) - return false; - Array VMCode(Length); - for (uint I=0;IFilters30.Count || FiltPos>OldFilterLengths.Count) - return false; - LastFilter=(int)FiltPos; - bool NewFilter=(FiltPos==Filters30.Count); - - UnpackFilter30 StackFilter=new UnpackFilter30(); // New filter for PrgStack. - - UnpackFilter30 Filter; - if (NewFilter) // New filter code, never used before since VM reset. - { - if (FiltPos>MAX3_UNPACK_FILTERS) - { - // Too many different filters, corrupt archive. - //delete StackFilter; - return false; - } - - Filters30.Add(1); - Filters30[Filters30.Count-1]=Filter=new UnpackFilter30(); - StackFilter.ParentFilter=(uint)(Filters30.Count-1); - - // Reserve one item to store the data block length of our new filter - // entry. We'll set it to real block length below, after reading it. - // But we need to initialize it now, because when processing corrupt - // data, we can access this item even before we set it to real value. - OldFilterLengths.Add(0); - } - else // Filter was used in the past. - { - Filter=Filters30[(int)FiltPos]; - StackFilter.ParentFilter=FiltPos; - } - - int EmptyCount=0; - for (int I=0;I0) - PrgStack[I]=null; - } - if (EmptyCount==0) - { - if (PrgStack.Count>MAX3_UNPACK_FILTERS) - { - //delete StackFilter; - return false; - } - PrgStack.Add(1); - EmptyCount=1; - } - size_t StackPos=(uint)(this.PrgStack.Count-EmptyCount); - PrgStack[(int)StackPos]=StackFilter; - - uint BlockStart=RarVM.ReadData(VMCodeInp); - if ((FirstByte & 0x40)!=0) - BlockStart+=258; - StackFilter.BlockStart=(uint)((BlockStart+UnpPtr)&MaxWinMask); - if ((FirstByte & 0x20)!=0) - { - StackFilter.BlockLength=RarVM.ReadData(VMCodeInp); - - // Store the last data block length for current filter. - OldFilterLengths[(int)FiltPos]=(int)StackFilter.BlockLength; - } - else - { - // Set the data block size to same value as the previous block size - // for same filter. It is possible for corrupt data to access a new - // and not filled yet item of OldFilterLengths array here. This is why - // we set new OldFilterLengths items to zero above. - StackFilter.BlockLength=FiltPos>9; - VMCodeInp.faddbits(7); - for (int I=0;I<7;I++) - if ((InitMask & (1<=0x10000 || VMCodeSize==0) - return false; - Array VMCode(VMCodeSize); - for (uint I=0;I>8; - VMCodeInp.faddbits(8); - } - VM.Prepare(&VMCode[0],VMCodeSize,&Filter->Prg); - } - StackFilter.Prg.Type=Filter.Prg.Type; - - return true; -} - - -bool UnpReadBuf30() -{ - int DataSize=ReadTop-Inp.InAddr; // Data left to process. - if (DataSize<0) - return false; - if (Inp.InAddr>BitInput.MAX_SIZE/2) - { - // If we already processed more than half of buffer, let's move - // remaining data into beginning to free more space for new data - // and ensure that calling function does not cross the buffer border - // even if we did not read anything here. Also it ensures that read size - // is not less than CRYPT_BLOCK_SIZE, so we can align it without risk - // to make it zero. - if (DataSize>0) - //x memmove(Inp.InBuf,Inp.InBuf+Inp.InAddr,DataSize); - Array.Copy(Inp.InBuf,Inp.InAddr,Inp.InBuf,0,DataSize); - Inp.InAddr=0; - ReadTop=DataSize; - } - else - DataSize=ReadTop; - int ReadCode=UnpIO_UnpRead(Inp.InBuf,DataSize,BitInput.MAX_SIZE-DataSize); - if (ReadCode>0) - ReadTop+=ReadCode; - ReadBorder=ReadTop-30; - return ReadCode!=-1; -} - -async System.Threading.Tasks.Task UnpReadBuf30Async(System.Threading.CancellationToken cancellationToken = default) -{ - int DataSize=ReadTop-Inp.InAddr; // Data left to process. - if (DataSize<0) - return false; - if (Inp.InAddr>BitInput.MAX_SIZE/2) - { - if (DataSize>0) - Array.Copy(Inp.InBuf,Inp.InAddr,Inp.InBuf,0,DataSize); - Inp.InAddr=0; - ReadTop=DataSize; - } - else - DataSize=ReadTop; - int ReadCode=await UnpIO_UnpReadAsync(Inp.InBuf,DataSize,BitInput.MAX_SIZE-DataSize,cancellationToken).ConfigureAwait(false); - if (ReadCode>0) - ReadTop+=ReadCode; - ReadBorder=ReadTop-30; - return ReadCode!=-1; -} - - -void UnpWriteBuf30() -{ - uint WrittenBorder=(uint)WrPtr; - uint WriteSize=(uint)((UnpPtr-WrittenBorder)&MaxWinMask); - for (int I=0;IParentFilter]->Prg; - VM_PreparedProgram *Prg=&flt->Prg; - - ExecuteCode(Prg); - - byte[] FilteredData=Prg.FilteredData; - uint FilteredDataSize=Prg.FilteredDataSize; - - delete PrgStack[I]; - PrgStack[I]=null; - while (I+1Prg; - VM_PreparedProgram *NextPrg=&NextFilter->Prg; - - ExecuteCode(NextPrg); - - FilteredData=NextPrg.FilteredData; - FilteredDataSize=NextPrg.FilteredDataSize; - I++; - delete PrgStack[I]; - PrgStack[I]=null; - } - UnpIO_UnpWrite(FilteredData,0,FilteredDataSize); - UnpSomeRead=true; - WrittenFileSize+=FilteredDataSize; - WrittenBorder=BlockEnd; - WriteSize=(uint)((UnpPtr-WrittenBorder)&MaxWinMask); - } - else - { - // Current filter intersects the window write border, so we adjust - // the window border to process this filter next time, not now. - for (size_t J=I;JInitR[6]=(uint)WrittenFileSize; - VM.Execute(Prg); -} - - -bool ReadTables30() -{ - byte[] BitLength = new byte[BC]; - byte[] Table = new byte[HUFF_TABLE_SIZE30]; - if (Inp.InAddr>ReadTop-25) - if (!UnpReadBuf30()) - return(false); - Inp.faddbits((uint)((8-Inp.InBit)&7)); - uint BitField=Inp.fgetbits(); - if ((BitField & 0x8000) != 0) - { - UnpBlockType=BLOCK_PPM; - return(PPM.DecodeInit(this,PPMEscChar)); - } - UnpBlockType=BLOCK_LZ; - - PrevLowDist=0; - LowDistRepCount=0; - - if ((BitField & 0x4000) == 0) - Utility.Memset(UnpOldTable,0,UnpOldTable.Length); - Inp.faddbits(2); - - for (uint I=0;I> 12); - Inp.faddbits(4); - if (Length==15) - { - uint ZeroCount=(byte)(Inp.fgetbits() >> 12); - Inp.faddbits(4); - if (ZeroCount==0) - BitLength[I]=15; - else - { - ZeroCount+=2; - while (ZeroCount-- > 0 && IReadTop-5) - if (!UnpReadBuf30()) - return(false); - uint Number=DecodeNumber(Inp,BlockTables.BD); - if (Number<16) - { - Table[I]=(byte)((Number+this.UnpOldTable[I]) & 0xf); - I++; - } - else - if (Number<18) - { - uint N; - if (Number==16) - { - N=(Inp.fgetbits() >> 13)+3; - Inp.faddbits(3); - } - else - { - N=(Inp.fgetbits() >> 9)+11; - Inp.faddbits(7); - } - if (I==0) - return false; // We cannot have "repeat previous" code at the first position. - else - while (N-- > 0 && I> 13)+3; - Inp.faddbits(3); - } - else - { - N=(Inp.fgetbits() >> 9)+11; - Inp.faddbits(7); - } - while (N-- > 0 && IReadTop) - return false; - MakeDecodeTables(Table,0,BlockTables.LD,NC30); - MakeDecodeTables(Table,(int)NC30,BlockTables.DD,DC30); - MakeDecodeTables(Table,(int)(NC30+DC30),BlockTables.LDD,LDC30); - MakeDecodeTables(Table,(int)(NC30+DC30+LDC30),BlockTables.RD,RC30); - //x memcpy(UnpOldTable,Table,sizeof(UnpOldTable)); - Array.Copy(Table,0,UnpOldTable,0,UnpOldTable.Length); - return true; -} - -#endif - -void UnpInitData30(bool Solid) -{ - if (!Solid) - { - TablesRead3=false; - Utility.Memset(UnpOldTable, 0, UnpOldTable.Length); - PPMEscChar=2; - UnpBlockType=BLOCK_LZ; - } - InitFilters30(Solid); -} - - -void InitFilters30(bool Solid) -{ - if (!Solid) - { - //OldFilterLengths.SoftReset(); - OldFilterLengths.Clear(); - LastFilter=0; - - //for (size_t I=0;I Date: Wed, 29 Oct 2025 10:23:57 +0000 Subject: [PATCH 18/25] ran out of tokens, things work but need one more conversion --- .../Compressors/Rar/UnpackV1/Unpack.cs | 292 +++++++++++++++++- 1 file changed, 291 insertions(+), 1 deletion(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index ac935576d..0fcf1d0e4 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -230,7 +230,7 @@ public async System.Threading.Tasks.Task DoUnpackAsync( case 29: // rar 3.x compression case 36: // alternative hash - Unpack29(fileHeader.IsSolid); + await Unpack29Async(fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); break; case 50: // rar 5.x compression @@ -558,6 +558,284 @@ private void Unpack29(bool solid) UnpWriteBuf(); } + private async System.Threading.Tasks.Task Unpack29Async( + bool solid, + System.Threading.CancellationToken cancellationToken = default + ) + { + int[] DDecode = new int[PackDef.DC]; + byte[] DBits = new byte[PackDef.DC]; + + int Bits; + + if (DDecode[1] == 0) + { + int Dist = 0, + BitLength = 0, + Slot = 0; + for (var I = 0; I < DBitLengthCounts.Length; I++, BitLength++) + { + var count = DBitLengthCounts[I]; + for (var J = 0; J < count; J++, Slot++, Dist += (1 << BitLength)) + { + DDecode[Slot] = Dist; + DBits[Slot] = (byte)BitLength; + } + } + } + + FileExtracted = true; + + if (!suspended) + { + UnpInitData(solid); + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + return; + } + if ( + (!solid || !tablesRead) + && ! ReadTables() + ) + { + return; + } + } + + if (ppmError) + { + return; + } + + while (true) + { + unpPtr &= PackDef.MAXWINMASK; + + if (inAddr > readBorder) + { + if (!await unpReadBufAsync(cancellationToken).ConfigureAwait(false)) + { + break; + } + } + + if (((wrPtr - unpPtr) & PackDef.MAXWINMASK) < 260 && wrPtr != unpPtr) + { + await UnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + if (destUnpSize < 0) + { + return; + } + if (suspended) + { + FileExtracted = false; + return; + } + } + if (unpBlockType == BlockTypes.BLOCK_PPM) + { + var ch = ppm.DecodeChar(); + if (ch == -1) + { + ppmError = true; + break; + } + if (ch == PpmEscChar) + { + var nextCh = ppm.DecodeChar(); + if (nextCh == 0) + { + if (! ReadTables()) + { + break; + } + continue; + } + if (nextCh == 2 || nextCh == -1) + { + break; + } + if (nextCh == 3) + { + if (!ReadVMCode()) + { + break; + } + continue; + } + if (nextCh == 4) + { + uint Distance = 0, + Length = 0; + var failed = false; + for (var I = 0; I < 4 && !failed; I++) + { + var ch2 = ppm.DecodeChar(); + if (ch2 == -1) + { + failed = true; + } + else if (I == 3) + { + Length = (uint)ch2; + } + else + { + Distance = (Distance << 8) + (uint)ch2; + } + } + if (failed) + { + break; + } + + CopyString(Length + 32, Distance + 2); + continue; + } + if (nextCh == 5) + { + var length = ppm.DecodeChar(); + if (length == -1) + { + break; + } + CopyString((uint)(length + 4), 1); + continue; + } + } + window[unpPtr++] = (byte)ch; + continue; + } + + var Number = this.decodeNumber(LD); + if (Number < 256) + { + window[unpPtr++] = (byte)Number; + continue; + } + if (Number >= 271) + { + var Length = LDecode[Number -= 271] + 3; + if ((Bits = LBits[Number]) > 0) + { + Length += GetBits() >> (16 - Bits); + AddBits(Bits); + } + + var DistNumber = this.decodeNumber(DD); + var Distance = DDecode[DistNumber] + 1; + if ((Bits = DBits[DistNumber]) > 0) + { + if (DistNumber > 9) + { + if (Bits > 4) + { + Distance += (GetBits() >> (20 - Bits)) << 4; + AddBits(Bits - 4); + } + if (lowDistRepCount > 0) + { + lowDistRepCount--; + Distance += prevLowDist; + } + else + { + var LowDist = this.decodeNumber(LDD); + if (LowDist == 16) + { + lowDistRepCount = PackDef.LOW_DIST_REP_COUNT - 1; + Distance += prevLowDist; + } + else + { + Distance += LowDist; + prevLowDist = (int)LowDist; + } + } + } + else + { + Distance += GetBits() >> (16 - Bits); + AddBits(Bits); + } + } + + if (Distance >= 0x2000) + { + Length++; + if (Distance >= 0x40000) + { + Length++; + } + } + + InsertOldDist(Distance); + lastLength = Length; + CopyString(Length, Distance); + continue; + } + if (Number == 256) + { + if (! ReadEndOfBlock()) + { + break; + } + continue; + } + if (Number == 257) + { + if (! ReadVMCode()) + { + break; + } + continue; + } + if (Number == 258) + { + if (lastLength != 0) + { + CopyString(lastLength, oldDist[0]); + } + + continue; + } + if (Number < 263) + { + var DistNum = Number - 259; + var Distance = (uint)oldDist[DistNum]; + for (var I = DistNum; I > 0; I--) + { + oldDist[I] = oldDist[I - 1]; + } + oldDist[0] = (int)Distance; + + var LengthNumber = this.decodeNumber(RD); + var Length = LDecode[LengthNumber] + 2; + if ((Bits = LBits[LengthNumber]) > 0) + { + Length += GetBits() >> (16 - Bits); + AddBits(Bits); + } + lastLength = Length; + CopyString((uint)Length, Distance); + continue; + } + if (Number < 272) + { + var Distance = SDDecode[Number -= 263] + 1; + if ((Bits = SDBits[Number]) > 0) + { + Distance += GetBits() >> (16 - Bits); + AddBits(Bits); + } + InsertOldDist((uint)Distance); + lastLength = 2; + CopyString(2, (uint)Distance); + } + } + await UnpWriteBufAsync(cancellationToken).ConfigureAwait(false); + } + private void UnpWriteBuf() { var WrittenBorder = wrPtr; @@ -1414,6 +1692,18 @@ private void ExecuteCode(VMPreparedProgram Prg) } } + + private async System.Threading.Tasks.Task UnpWriteBufAsync( + System.Threading.CancellationToken cancellationToken = default + ) + { + // UnpWriteBuf writes to the output stream + // For full async, this needs to use writeStream.WriteAsync + // For now, call synchronous version + UnpWriteBuf(); + await System.Threading.Tasks.Task.CompletedTask; + } + private void CleanUp() { if (ppm != null) From dc31e4c5fad1472dc4982d6e036d4436b25daf8b Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 10:27:30 +0000 Subject: [PATCH 19/25] fmt --- src/SharpCompress/Archives/Rar/RarArchiveEntry.cs | 1 - src/SharpCompress/Compressors/Rar/RarStream.cs | 6 +++--- src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs | 12 ++++-------- .../Compressors/Rar/UnpackV2017/Unpack.cs | 7 ++++++- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 55818b79b..aaba6d1ec 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -105,7 +105,6 @@ public async Task OpenEntryStreamAsync(CancellationToken cancellationTok } else { - stream = new RarStream( archive.UnpackV2017.Value, FileHeader, diff --git a/src/SharpCompress/Compressors/Rar/RarStream.cs b/src/SharpCompress/Compressors/Rar/RarStream.cs index 6bdb31545..ca718d616 100644 --- a/src/SharpCompress/Compressors/Rar/RarStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarStream.cs @@ -149,8 +149,7 @@ public override async System.Threading.Tasks.Task ReadAsync( int offset, int count, System.Threading.CancellationToken cancellationToken - ) => - await ReadImplAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ) => await ReadImplAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); private async System.Threading.Tasks.Task ReadImplAsync( byte[] buffer, @@ -200,7 +199,8 @@ public override async System.Threading.Tasks.ValueTask ReadAsync( var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); try { - var bytesRead =await ReadImplAsync(array, 0, buffer.Length, cancellationToken).ConfigureAwait(false); + var bytesRead = await ReadImplAsync(array, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); new ReadOnlySpan(array, 0, bytesRead).CopyTo(buffer.Span); return bytesRead; } diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index 0fcf1d0e4..25f026f15 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -593,10 +593,7 @@ private async System.Threading.Tasks.Task Unpack29Async( { return; } - if ( - (!solid || !tablesRead) - && ! ReadTables() - ) + if ((!solid || !tablesRead) && !ReadTables()) { return; } @@ -645,7 +642,7 @@ private async System.Threading.Tasks.Task Unpack29Async( var nextCh = ppm.DecodeChar(); if (nextCh == 0) { - if (! ReadTables()) + if (!ReadTables()) { break; } @@ -776,7 +773,7 @@ private async System.Threading.Tasks.Task Unpack29Async( } if (Number == 256) { - if (! ReadEndOfBlock()) + if (!ReadEndOfBlock()) { break; } @@ -784,7 +781,7 @@ private async System.Threading.Tasks.Task Unpack29Async( } if (Number == 257) { - if (! ReadVMCode()) + if (!ReadVMCode()) { break; } @@ -1692,7 +1689,6 @@ private void ExecuteCode(VMPreparedProgram Prg) } } - private async System.Threading.Tasks.Task UnpWriteBufAsync( System.Threading.CancellationToken cancellationToken = default ) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs index f466d8537..ea20a0fa1 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.cs @@ -115,7 +115,12 @@ public async System.Threading.Tasks.Task DoUnpackAsync( { // TODO: When compression methods are converted to async, call them here // For now, fall back to synchronous version - await DoUnpackAsync(fileHeader.CompressionAlgorithm, fileHeader.IsSolid, cancellationToken).ConfigureAwait(false); + await DoUnpackAsync( + fileHeader.CompressionAlgorithm, + fileHeader.IsSolid, + cancellationToken + ) + .ConfigureAwait(false); } } From f238be6003b7b2350a5457c3798ebef9d4fbdfa7 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 10:35:03 +0000 Subject: [PATCH 20/25] add arraypool usage in init --- .../Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index 6ee5463a5..d7890e940 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using SharpCompress.Common; using static SharpCompress.Compressors.Rar.UnpackV2017.PackDef; using static SharpCompress.Compressors.Rar.UnpackV2017.UnpackGlobal; @@ -110,7 +111,7 @@ private void Init(size_t WinSize, bool Solid) throw new InvalidFormatException("Grow && Fragmented"); } - var NewWindow = Fragmented ? null : new byte[WinSize]; + var NewWindow = Fragmented ? null : ArrayPool.Shared.Rent((int)WinSize); if (NewWindow == null) { @@ -126,6 +127,7 @@ private void Init(size_t WinSize, bool Solid) if (Window != null) // If allocated by preceding files. { //free(Window); + ArrayPool.Shared.Return(Window); Window = null; } FragWindow.Init(WinSize); From 65e607454e73300d30c4cb78d97a3e1f9ae3153c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 13:07:43 +0000 Subject: [PATCH 21/25] add async for UnpWriteBufAsync and remove comments --- .../Compressors/Rar/UnpackV1/Unpack.cs | 249 +++++++++++++++++- .../Rar/UnpackV2017/Unpack.unpack_cpp.cs | 27 -- 2 files changed, 244 insertions(+), 32 deletions(-) diff --git a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs index 25f026f15..f59fed9e0 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs @@ -1693,11 +1693,250 @@ private async System.Threading.Tasks.Task UnpWriteBufAsync( System.Threading.CancellationToken cancellationToken = default ) { - // UnpWriteBuf writes to the output stream - // For full async, this needs to use writeStream.WriteAsync - // For now, call synchronous version - UnpWriteBuf(); - await System.Threading.Tasks.Task.CompletedTask; + var WrittenBorder = wrPtr; + var WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK; + for (var I = 0; I < prgStack.Count; I++) + { + var flt = prgStack[I]; + if (flt is null) + { + continue; + } + if (flt.NextWindow) + { + flt.NextWindow = false; + continue; + } + var BlockStart = flt.BlockStart; + var BlockLength = flt.BlockLength; + if (((BlockStart - WrittenBorder) & PackDef.MAXWINMASK) < WriteSize) + { + if (WrittenBorder != BlockStart) + { + await UnpWriteAreaAsync(WrittenBorder, BlockStart, cancellationToken) + .ConfigureAwait(false); + WrittenBorder = BlockStart; + WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK; + } + if (BlockLength <= WriteSize) + { + var BlockEnd = (BlockStart + BlockLength) & PackDef.MAXWINMASK; + if (BlockStart < BlockEnd || BlockEnd == 0) + { + rarVM.setMemory(0, window, BlockStart, BlockLength); + } + else + { + var FirstPartLength = PackDef.MAXWINSIZE - BlockStart; + rarVM.setMemory(0, window, BlockStart, FirstPartLength); + rarVM.setMemory(FirstPartLength, window, 0, BlockEnd); + } + + var ParentPrg = filters[flt.ParentFilter].Program; + var Prg = flt.Program; + + if (ParentPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE) + { + Prg.GlobalData.Clear(); + for ( + var i = 0; + i < ParentPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE; + i++ + ) + { + Prg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = ParentPrg.GlobalData[ + RarVM.VM_FIXEDGLOBALSIZE + i + ]; + } + } + + ExecuteCode(Prg); + + if (Prg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE) + { + if (ParentPrg.GlobalData.Count < Prg.GlobalData.Count) + { + ParentPrg.GlobalData.SetSize(Prg.GlobalData.Count); + } + + for (var i = 0; i < Prg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE; i++) + { + ParentPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = Prg.GlobalData[ + RarVM.VM_FIXEDGLOBALSIZE + i + ]; + } + } + else + { + ParentPrg.GlobalData.Clear(); + } + + var FilteredDataOffset = Prg.FilteredDataOffset; + var FilteredDataSize = Prg.FilteredDataSize; + var FilteredData = ArrayPool.Shared.Rent(FilteredDataSize); + try + { + Array.Copy( + rarVM.Mem, + FilteredDataOffset, + FilteredData, + 0, + FilteredDataSize + ); + + prgStack[I] = null; + while (I + 1 < prgStack.Count) + { + var NextFilter = prgStack[I + 1]; + if ( + NextFilter is null + || NextFilter.BlockStart != BlockStart + || NextFilter.BlockLength != FilteredDataSize + || NextFilter.NextWindow + ) + { + break; + } + + rarVM.setMemory(0, FilteredData, 0, FilteredDataSize); + + var pPrg = filters[NextFilter.ParentFilter].Program; + var NextPrg = NextFilter.Program; + + if (pPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE) + { + NextPrg.GlobalData.SetSize(pPrg.GlobalData.Count); + + for ( + var i = 0; + i < pPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE; + i++ + ) + { + NextPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = + pPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i]; + } + } + + ExecuteCode(NextPrg); + + if (NextPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE) + { + if (pPrg.GlobalData.Count < NextPrg.GlobalData.Count) + { + pPrg.GlobalData.SetSize(NextPrg.GlobalData.Count); + } + + for ( + var i = 0; + i < NextPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE; + i++ + ) + { + pPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = + NextPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i]; + } + } + else + { + pPrg.GlobalData.Clear(); + } + + FilteredDataOffset = NextPrg.FilteredDataOffset; + FilteredDataSize = NextPrg.FilteredDataSize; + if (FilteredData.Length < FilteredDataSize) + { + ArrayPool.Shared.Return(FilteredData); + FilteredData = ArrayPool.Shared.Rent(FilteredDataSize); + } + for (var i = 0; i < FilteredDataSize; i++) + { + FilteredData[i] = NextPrg.GlobalData[FilteredDataOffset + i]; + } + + I++; + prgStack[I] = null; + } + + await writeStream + .WriteAsync(FilteredData, 0, FilteredDataSize, cancellationToken) + .ConfigureAwait(false); + writtenFileSize += FilteredDataSize; + destUnpSize -= FilteredDataSize; + WrittenBorder = BlockEnd; + WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK; + } + finally + { + ArrayPool.Shared.Return(FilteredData); + } + } + else + { + for (var J = I; J < prgStack.Count; J++) + { + var filt = prgStack[J]; + if (filt != null && filt.NextWindow) + { + filt.NextWindow = false; + } + } + wrPtr = WrittenBorder; + return; + } + } + } + + await UnpWriteAreaAsync(WrittenBorder, unpPtr, cancellationToken).ConfigureAwait(false); + wrPtr = unpPtr; + } + + private async System.Threading.Tasks.Task UnpWriteAreaAsync( + int startPtr, + int endPtr, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (endPtr < startPtr) + { + await UnpWriteDataAsync( + window, + startPtr, + -startPtr & PackDef.MAXWINMASK, + cancellationToken + ) + .ConfigureAwait(false); + await UnpWriteDataAsync(window, 0, endPtr, cancellationToken).ConfigureAwait(false); + } + else + { + await UnpWriteDataAsync(window, startPtr, endPtr - startPtr, cancellationToken) + .ConfigureAwait(false); + } + } + + private async System.Threading.Tasks.Task UnpWriteDataAsync( + byte[] data, + int offset, + int size, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (destUnpSize < 0) + { + return; + } + var writeSize = size; + if (writeSize > destUnpSize) + { + writeSize = (int)destUnpSize; + } + await writeStream + .WriteAsync(data, offset, writeSize, cancellationToken) + .ConfigureAwait(false); + + writtenFileSize += size; + destUnpSize -= size; } private void CleanUp() diff --git a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs index d7890e940..0fe817a65 100644 --- a/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs +++ b/src/SharpCompress/Compressors/Rar/UnpackV2017/Unpack.unpack_cpp.cs @@ -5,13 +5,7 @@ using SharpCompress.Common; using static SharpCompress.Compressors.Rar.UnpackV2017.PackDef; using static SharpCompress.Compressors.Rar.UnpackV2017.UnpackGlobal; -#if !Rar2017_64bit using size_t = System.UInt32; -#else -using nint = System.Int64; -using nuint = System.UInt64; -using size_t = System.UInt64; -#endif namespace SharpCompress.Compressors.Rar.UnpackV2017; @@ -43,11 +37,9 @@ public Unpack( /* ComprDataIO *DataIO */ // It prevents crash if first DoUnpack call is later made with wrong // (true) 'Solid' value. UnpInitData(false); -#if !RarV2017_SFX_MODULE // RAR 1.5 decompression initialization UnpInitData15(false); InitHuff(); -#endif } // later: may need Dispose() if we support thread pool @@ -172,7 +164,6 @@ private void DoUnpack(uint Method, bool Solid) // just for extra safety. switch (Method) { -#if !RarV2017_SFX_MODULE case 15: // rar 1.5 compression if (!Fragmented) { @@ -188,8 +179,6 @@ private void DoUnpack(uint Method, bool Solid) } break; -#endif -#if !RarV2017_RAR5ONLY case 29: // rar 3.x compression if (!Fragmented) { @@ -197,23 +186,7 @@ private void DoUnpack(uint Method, bool Solid) } break; -#endif case 50: // RAR 5.0 compression algorithm. - /*#if RarV2017_RAR_SMP - if (MaxUserThreads > 1) - { - // We do not use the multithreaded unpack routine to repack RAR archives - // in 'suspended' mode, because unlike the single threaded code it can - // write more than one dictionary for same loop pass. So we would need - // larger buffers of unknown size. Also we do not support multithreading - // in fragmented window mode. - if (!Fragmented) - { - Unpack5MT(Solid); - break; - } - } - #endif*/ Unpack5(Solid); break; #if !Rar2017_NOSTRICT From 75ada5623cc5ee2cf1e8a27f77561ead0b9e28ad Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 13:23:50 +0000 Subject: [PATCH 22/25] add async tests for compress stream --- src/SharpCompress/IO/SharpCompressStream.cs | 2 - .../Streams/SharpCompressStreamAsyncTests.cs | 175 ++++++++++++++++++ 2 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 tests/SharpCompress.Test/Streams/SharpCompressStreamAsyncTests.cs diff --git a/src/SharpCompress/IO/SharpCompressStream.cs b/src/SharpCompress/IO/SharpCompressStream.cs index 238112654..1dadce4e8 100644 --- a/src/SharpCompress/IO/SharpCompressStream.cs +++ b/src/SharpCompress/IO/SharpCompressStream.cs @@ -90,8 +90,6 @@ void IStreamStack.SetPosition(long position) { } public Stream Stream { get; } - //private MemoryStream _bufferStream = new(); - private bool _readOnly; //some archive detection requires seek to be disabled to cause it to exception to try the next arc type //private bool _isRewound; diff --git a/tests/SharpCompress.Test/Streams/SharpCompressStreamAsyncTests.cs b/tests/SharpCompress.Test/Streams/SharpCompressStreamAsyncTests.cs new file mode 100644 index 000000000..a33560a6e --- /dev/null +++ b/tests/SharpCompress.Test/Streams/SharpCompressStreamAsyncTests.cs @@ -0,0 +1,175 @@ +using System; +using System.Buffers; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SharpCompress.Compressors.LZMA; +using SharpCompress.IO; +using Xunit; + +namespace SharpCompress.Test.Streams; + +public class SharpCompressStreamAsyncTests +{ + private static void CreateData(MemoryStream ms) + { + using (BinaryWriter bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + // write offset every 4 bytes - easy to test position + for (int i = 0; i < ms.Length; i += 4) + { + bw.Write(i); + } + } + ms.Position = 0; + } + + [Fact] + public async Task BufferReadAsyncTest() + { + byte[] data = ArrayPool.Shared.Rent(0x100000); + byte[] test = ArrayPool.Shared.Rent(0x1000); + try + { + using (MemoryStream ms = new MemoryStream(data)) + { + CreateData(ms); + + using (SharpCompressStream scs = new SharpCompressStream(ms, true, false, 0x10000)) + { + IStreamStack stack = (IStreamStack)scs; + + scs.Seek(0x1000, SeekOrigin.Begin); + Assert.Equal(0x1000, scs.Position); // position in the SharpCompressionStream + Assert.Equal(0x1000, ms.Position); // initial seek + full buffer read + + await scs.ReadAsync(test, 0, test.Length); // read bytes 0x1000 to 0x2000 + Assert.Equal(0x2000, scs.Position); // stream has correct position + Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); // is the data correct + Assert.Equal(0x11000, ms.Position); // seek plus read bytes + + scs.Seek(0x500, SeekOrigin.Begin); // seek before the buffer start + await scs.ReadAsync(test, 0, test.Length); // read bytes 0x500 to 0x1500 + Assert.Equal(0x1500, scs.Position); // stream has correct position + Assert.True(data.Skip(0x500).Take(test.Length).SequenceEqual(test)); // is the data correct + Assert.Equal(0x10500, ms.Position); // seek plus read bytes + } + } + } + finally + { + ArrayPool.Shared.Return(data); + ArrayPool.Shared.Return(test); + } + } + + [Fact] + public async Task BufferReadAndSeekAsyncTest() + { + byte[] data = ArrayPool.Shared.Rent(0x100000); + byte[] test = ArrayPool.Shared.Rent(0x1000); + try + { + using (MemoryStream ms = new MemoryStream(data)) + { + CreateData(ms); + + using (SharpCompressStream scs = new SharpCompressStream(ms, true, false, 0x10000)) + { + IStreamStack stack = (IStreamStack)scs; + + await scs.ReadAsync(test, 0, test.Length); // read bytes 0 to 0x1000 + Assert.True(data.Take(test.Length).SequenceEqual(test)); // is the data correct + Assert.Equal(0x1000, scs.Position); // stream has correct position + Assert.Equal(0x10000, ms.Position); // moved the base stream on by buffer size + + await scs.ReadAsync(test, 0, test.Length); // read bytes 0x1000 to 0x2000 + Assert.Equal(0x2000, scs.Position); // stream has correct position + Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); // is the data correct + Assert.Equal(0x10000, ms.Position); // the base stream has not moved + + // rewind the buffer + stack.Rewind(0x1000); // rewind buffer back by 0x1000 bytes + + // repeat the previous test + await scs.ReadAsync(test, 0, test.Length); // read bytes 0x1000 to 0x2000 + Assert.Equal(0x2000, scs.Position); // stream has correct position + Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); // is the data correct + Assert.Equal(0x10000, ms.Position); // the base stream has not moved + } + } + } + finally + { + ArrayPool.Shared.Return(data); + ArrayPool.Shared.Return(test); + } + } + + [Fact] + public async Task MultipleAsyncReadsTest() + { + byte[] data = ArrayPool.Shared.Rent(0x100000); + byte[] test1 = ArrayPool.Shared.Rent(0x800); + byte[] test2 = ArrayPool.Shared.Rent(0x800); + try + { + using (MemoryStream ms = new MemoryStream(data)) + { + CreateData(ms); + + using (SharpCompressStream scs = new SharpCompressStream(ms, true, false, 0x10000)) + { + // Read first chunk + await scs.ReadAsync(test1, 0, test1.Length); + Assert.Equal(0x800, scs.Position); + Assert.True(data.Take(test1.Length).SequenceEqual(test1)); // first read is correct + + // Read second chunk + await scs.ReadAsync(test2, 0, test2.Length); + Assert.Equal(0x1000, scs.Position); + Assert.True(data.Skip(test1.Length).Take(test2.Length).SequenceEqual(test2)); // second read is correct + } + } + } + finally + { + ArrayPool.Shared.Return(data); + ArrayPool.Shared.Return(test1); + ArrayPool.Shared.Return(test2); + } + } + + [Fact] + public async Task LargeBufferAsyncReadTest() + { + byte[] data = ArrayPool.Shared.Rent(0x200000); + byte[] test = ArrayPool.Shared.Rent(0x8000); + try + { + using (MemoryStream ms = new MemoryStream(data)) + { + CreateData(ms); + + using (SharpCompressStream scs = new SharpCompressStream(ms, true, false, 0x10000)) + { + for (int i = 0; i < 10; i++) + { + await scs.ReadAsync(test, 0, test.Length); + long expectedPosition = (long)(i + 1) * test.Length; + Assert.Equal(expectedPosition, scs.Position); + Assert.True( + data.Skip(i * test.Length).Take(test.Length).SequenceEqual(test) + ); + } + } + } + } + finally + { + ArrayPool.Shared.Return(data); + ArrayPool.Shared.Return(test); + } + } +} From e786e953585b60d1380eb27d04ba58d788fd83bd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:25:48 +0000 Subject: [PATCH 23/25] Initial plan From 88b3a66bf931b021ac80175d635c57bcf0fa0904 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:40:44 +0000 Subject: [PATCH 24/25] Fix Windows test failures in SharpCompressStreamTests ArrayPool.Rent() can return buffers larger than requested. The tests were using test.Length (the actual buffer size) instead of the requested size (0x1000), causing failures on Windows where ArrayPool returns larger buffers than on Linux. Fixed by: - Using explicit size (0x1000) instead of test.Length in Read() calls - Using test.Take(0x1000) instead of test when comparing arrays Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- src/SharpCompress/packages.lock.json | 6 +++--- .../Streams/SharpCompressStreamTest.cs | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index 9ebb16774..b85a38f73 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -335,9 +335,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.17, )", - "resolved": "8.0.17", - "contentHash": "x5/y4l8AtshpBOrCZdlE4txw8K3e3s9meBFeZeR3l8hbbku2V7kK6ojhXvrbjg1rk3G+JqL1BI26gtgc1ZrdUw==" + "requested": "[8.0.20, )", + "resolved": "8.0.20", + "contentHash": "Rhcto2AjGvTO62+/VTmBpumBOmqIGp7nYEbTbmEXkCq4yPGxV8whju3/HsIA/bKyo2+DggaYk5+/8sxb1AbPTw==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", diff --git a/tests/SharpCompress.Test/Streams/SharpCompressStreamTest.cs b/tests/SharpCompress.Test/Streams/SharpCompressStreamTest.cs index e664231c4..5161076bb 100644 --- a/tests/SharpCompress.Test/Streams/SharpCompressStreamTest.cs +++ b/tests/SharpCompress.Test/Streams/SharpCompressStreamTest.cs @@ -41,15 +41,15 @@ public void BufferReadTest() Assert.Equal(0x1000, scs.Position); //position in the SharpCompressionStream (with 0xf000 remaining in the buffer) Assert.Equal(0x1000, ms.Position); //initial seek + full buffer read - scs.Read(test, 0, test.Length); //read bytes 0x1000 to 0x2000 + scs.Read(test, 0, 0x1000); //read bytes 0x1000 to 0x2000 Assert.Equal(0x2000, scs.Position); //stream has correct position - Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); //is the data correct + Assert.True(data.Skip(0x1000).Take(0x1000).SequenceEqual(test.Take(0x1000))); //is the data correct Assert.Equal(0x11000, ms.Position); //seek plus read bytes scs.Seek(0x500, SeekOrigin.Begin); //seek before the buffer start - scs.Read(test, 0, test.Length); //read bytes 0x500 to 0x1500 + scs.Read(test, 0, 0x1000); //read bytes 0x500 to 0x1500 Assert.Equal(0x1500, scs.Position); //stream has correct position - Assert.True(data.Skip(0x500).Take(test.Length).SequenceEqual(test)); //is the data correct + Assert.True(data.Skip(0x500).Take(0x1000).SequenceEqual(test.Take(0x1000))); //is the data correct Assert.Equal(0x10500, ms.Position); //seek plus read bytes } } @@ -71,23 +71,23 @@ public void BufferReadAndSeekTest() { IStreamStack stack = (IStreamStack)scs; - scs.Read(test, 0, test.Length); //read bytes 0 to 0x1000 - Assert.True(data.Take(test.Length).SequenceEqual(test)); //is the data correct + scs.Read(test, 0, 0x1000); //read bytes 0 to 0x1000 + Assert.True(data.Take(0x1000).SequenceEqual(test.Take(0x1000))); //is the data correct Assert.Equal(0x1000, scs.Position); //stream has correct position Assert.Equal(0x10000, ms.Position); //moved the base stream on by the size of the buffer not what was requested - scs.Read(test, 0, test.Length); //read bytes 0x1000 to 0x2000 + scs.Read(test, 0, 0x1000); //read bytes 0x1000 to 0x2000 Assert.Equal(0x2000, scs.Position); //stream has correct position - Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); //is the data correct + Assert.True(data.Skip(0x1000).Take(0x1000).SequenceEqual(test.Take(0x1000))); //is the data correct Assert.Equal(0x10000, ms.Position); //the base stream has not moved //rewind the buffer stack.Rewind(0x1000); //rewind buffer back by 0x1000 bytes //repeat the previous test - scs.Read(test, 0, test.Length); //read bytes 0x1000 to 0x2000 + scs.Read(test, 0, 0x1000); //read bytes 0x1000 to 0x2000 Assert.Equal(0x2000, scs.Position); //stream has correct position - Assert.True(data.Skip(test.Length).Take(test.Length).SequenceEqual(test)); //is the data correct + Assert.True(data.Skip(0x1000).Take(0x1000).SequenceEqual(test.Take(0x1000))); //is the data correct Assert.Equal(0x10000, ms.Position); //the base stream has not moved } } From ab7196f86cbb576e0b7f15367caa818f2a1c5724 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Wed, 29 Oct 2025 14:07:12 +0000 Subject: [PATCH 25/25] some review fixes --- src/SharpCompress/Common/EntryStream.cs | 2 -- .../Compressors/Rar/RarCrcStream.cs | 32 +++++++------------ 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index 7ac2aa840..9e87e25e0 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -118,8 +118,6 @@ public override async ValueTask DisposeAsync() await lzmaStream.FlushAsync().ConfigureAwait(false); } } - - _isDisposed = true; #if DEBUG_STREAMS this.DebugDispose(typeof(EntryStream)); #endif diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 334a50b7a..0ec0527dd 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -136,30 +136,22 @@ public override async System.Threading.Tasks.ValueTask ReadAsync( ) { cancellationToken.ThrowIfCancellationRequested(); - var array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); - try + var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (result != 0) { - var result = await base.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); - if (result != 0) - { - currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span, 0, result); - } - else if ( - !disableCRC - && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) - && buffer.Length != 0 - ) - { - // NOTE: we use the last FileHeader in a multipart volume to check CRC - throw new InvalidFormatException("file crc mismatch"); - } - - return result; + currentCrc = RarCRC.CheckCrc(currentCrc, buffer.Span, 0, result); } - finally + else if ( + !disableCRC + && GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) + && buffer.Length != 0 + ) { - System.Buffers.ArrayPool.Shared.Return(array); + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); } + + return result; } #endif }