Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
116 commits
Select commit Hold shift + click to select a range
a410f73
archive asyncs are more right
adamhathcock Jan 15, 2026
2d597e6
be more lazy with loading of sync stuff
adamhathcock Jan 15, 2026
33b6447
Merge remote-tracking branch 'origin/master' into adam/async-creation
adamhathcock Jan 15, 2026
63736ef
Merge remote-tracking branch 'origin/master' into adam/async-creation
adamhathcock Jan 15, 2026
810df8a
revert lazy archive
adamhathcock Jan 15, 2026
5c06b8c
enable single test
adamhathcock Jan 15, 2026
0b2158f
Initial plan
Copilot Jan 15, 2026
bbba2e6
Initial plan for fixing SevenZipArchive_LZMA_AsyncStreamExtraction test
Copilot Jan 15, 2026
9bb670a
Fix SevenZipArchive async stream handling by adding async Open and Re…
Copilot Jan 15, 2026
491beab
uncomment tests
adamhathcock Jan 16, 2026
f4ce4cb
fix tests for both frameworks
adamhathcock Jan 16, 2026
394d982
Merge pull request #1133 from adamhathcock/copilot/sub-pr-1132
adamhathcock Jan 16, 2026
63e124e
Upgrade xunit to v3
adamhathcock Jan 16, 2026
75cc368
Initial plan
Copilot Jan 16, 2026
2906529
Fix ReaderFactory.OpenAsyncReader to use async IsArchiveAsync methods
Copilot Jan 16, 2026
e919930
Fix Archive async tests to not use AsyncOnlyStream (archives need see…
Copilot Jan 16, 2026
c7da19f
Format code with CSharpier
Copilot Jan 16, 2026
bbbbc88
Initial plan
Copilot Jan 16, 2026
b9fccbd
Update src/SharpCompress/Factories/ZStandardFactory.cs
adamhathcock Jan 16, 2026
302cf2e
Initial plan
Copilot Jan 16, 2026
2012077
Remove redundant _stream field from AsyncOnlyStream and use base Stre…
Copilot Jan 16, 2026
9fa686b
Fix empty catch blocks in TarArchive.Factory.cs with explicit excepti…
Copilot Jan 16, 2026
5336eb6
Merge pull request #1138 from adamhathcock/copilot/sub-pr-1132
adamhathcock Jan 16, 2026
3b10be5
Merge pull request #1140 from adamhathcock/copilot/sub-pr-1132-anothe…
adamhathcock Jan 16, 2026
0de5c59
Restore AsyncOnlyStream in archive async tests as requested
Copilot Jan 16, 2026
c958d18
Merge pull request #1137 from adamhathcock/copilot/sub-pr-1136
adamhathcock Jan 16, 2026
4b34dd6
Initial plan
Copilot Jan 16, 2026
f7c91bb
Update src/SharpCompress/Factories/SevenZipFactory.cs
adamhathcock Jan 16, 2026
4b9b20d
Initial plan
Copilot Jan 16, 2026
b0fde2b
Fix ReadFullyAsync call to specify offset and count for ArrayPool buffer
Copilot Jan 16, 2026
6e5e47f
Update SevenZipFactory to consistently call OpenAsyncArchive methods
Copilot Jan 16, 2026
d5f9815
Merge pull request #1136 from adamhathcock/adam/upgrade-xunit
adamhathcock Jan 16, 2026
29f4c7f
Merge pull request #1142 from adamhathcock/copilot/sub-pr-1132-anothe…
adamhathcock Jan 16, 2026
2180df3
Pass CancellationToken.None explicitly to OpenAsyncArchive methods
Copilot Jan 16, 2026
cfe59fc
Merge branch 'adam/async-creation' into copilot/sub-pr-1132
adamhathcock Jan 16, 2026
1cc80e7
Merge pull request #1141 from adamhathcock/copilot/sub-pr-1132
adamhathcock Jan 16, 2026
cc59c19
fix ace tests
adamhathcock Jan 16, 2026
ec7c359
Arj works
adamhathcock Jan 16, 2026
cd70a77
remvoe AutoFactory
adamhathcock Jan 16, 2026
763805e
async IsRarFile
adamhathcock Jan 16, 2026
447d352
some fixes
adamhathcock Jan 16, 2026
82d56b9
multi-file rars done manually
adamhathcock Jan 16, 2026
f99e421
fix factory
adamhathcock Jan 16, 2026
8e54b10
tar tests are better?
adamhathcock Jan 16, 2026
4c4b727
Tar detection works
adamhathcock Jan 17, 2026
408d2e6
Async add entry
adamhathcock Jan 18, 2026
08118f7
add more async writing
adamhathcock Jan 18, 2026
f359f55
some minor fixes
adamhathcock Jan 18, 2026
2e95832
factory the headers instead of creating
adamhathcock Jan 19, 2026
884f0b7
some grunt rar header async
adamhathcock Jan 19, 2026
ecd9317
more basic LLM async and fixed CRC async
adamhathcock Jan 19, 2026
44174e7
some fixes
adamhathcock Jan 20, 2026
b546172
more async fixes?
adamhathcock Jan 20, 2026
3b5ee48
fix for another async typo
adamhathcock Jan 20, 2026
05bf22f
rar works now
adamhathcock Jan 20, 2026
8abb972
Fix test
adamhathcock Jan 20, 2026
2175cb2
tar fixes
adamhathcock Jan 20, 2026
b26d38b
another tar test fix
adamhathcock Jan 20, 2026
3987733
LZW async
adamhathcock Jan 20, 2026
ff0769e
Create factory for CBZip2InputStream
adamhathcock Jan 20, 2026
0a9c5bf
format changes
adamhathcock Jan 20, 2026
a8d5b8e
intermediate commit
adamhathcock Jan 20, 2026
cc47fde
works?
adamhathcock Jan 20, 2026
4d3ae3a
Merge branch 'opencode/curious-river' into adam/bzip2-async
adamhathcock Jan 20, 2026
f6faaa8
better async bzip input stream
adamhathcock Jan 20, 2026
e1bbc65
more bzip tests pass
adamhathcock Jan 20, 2026
cf901c2
fix test
adamhathcock Jan 20, 2026
895699d
fmt
adamhathcock Jan 20, 2026
c38f74d
Merge remote-tracking branch 'origin/master' into adam/async-creation
adamhathcock Jan 21, 2026
169364f
fix disposal
adamhathcock Jan 21, 2026
7b7eba8
more fixes
adamhathcock Jan 21, 2026
8df9232
use extension where appropriate with more fixes
adamhathcock Jan 21, 2026
1a87075
GZip fix
adamhathcock Jan 22, 2026
85d82e5
fix tar issue
adamhathcock Jan 22, 2026
db0bb8a
fix some 7z tests
adamhathcock Jan 22, 2026
b9ed2b0
fmt
adamhathcock Jan 22, 2026
c5d7407
Update from Task to ValueTask where I can
adamhathcock Jan 22, 2026
a8f4723
divide async and sync into new files
adamhathcock Jan 22, 2026
61c6f84
some manual moving
adamhathcock Jan 22, 2026
d1f6fd9
move more and fmt
adamhathcock Jan 22, 2026
4c838db
everything compiles and passes (minus 3 tests)
adamhathcock Jan 22, 2026
65208a3
fix more tests
adamhathcock Jan 22, 2026
ae4f2c0
check if second stream is zip header without changing position - fix
adamhathcock Jan 22, 2026
5152e31
fix build flags
adamhathcock Jan 22, 2026
77c1ceb
Merge remote-tracking branch 'origin/master' into adam/async-creation
adamhathcock Jan 22, 2026
9403c12
Add await
adamhathcock Jan 22, 2026
b4f949b
Initial plan
Copilot Jan 22, 2026
336a8f2
Fix SharpCompressStream Dispose methods to set _isDisposed and call b…
Copilot Jan 22, 2026
d9be638
Address code review feedback - remove extra blank lines and use consi…
Copilot Jan 22, 2026
c581450
clean up and fixing tests....need to revisit disposal
adamhathcock Jan 22, 2026
b622a2c
fix disposal and other simple issues
adamhathcock Jan 22, 2026
3b83d08
fmt
adamhathcock Jan 22, 2026
16831e1
Merge pull request #1152 from adamhathcock/copilot/sub-pr-1132
adamhathcock Jan 22, 2026
11b92d1
Create for explodestream
adamhathcock Jan 22, 2026
4440241
LZMA create
adamhathcock Jan 22, 2026
fbc168f
Merge remote-tracking branch 'origin/adam/async-creation' into adam/a…
adamhathcock Jan 23, 2026
060b1ed
fix disposal and add tests
adamhathcock Jan 23, 2026
abe0087
fmt
adamhathcock Jan 23, 2026
414cad1
add braces
adamhathcock Jan 23, 2026
14fd880
add tar writing async
adamhathcock Jan 25, 2026
55100cb
ExplodeStream is async
adamhathcock Jan 25, 2026
e89fb21
gzipwriter async
adamhathcock Jan 25, 2026
86c3b93
Merge branch 'opencode/glowing-wolf' into adam/async-creation
adamhathcock Jan 25, 2026
73704bc
Merge branch 'opencode/clever-knight' into adam/async-creation
adamhathcock Jan 25, 2026
d0823db
fmt
adamhathcock Jan 25, 2026
def0bce
remove mono dep as it's annoying
adamhathcock Jan 25, 2026
244acc0
implemented async rangecoder
adamhathcock Jan 25, 2026
f364b68
remove more buffer
adamhathcock Jan 25, 2026
507074c
Merge branch 'opencode/glowing-wolf' into adam/async-creation
adamhathcock Jan 25, 2026
4d84394
LZMA Lencoder uses async
adamhathcock Jan 25, 2026
984ea8f
remove posix
adamhathcock Jan 25, 2026
04eabb7
Merge remote-tracking branch 'origin/master' into adam/async-creation
adamhathcock Jan 26, 2026
979c8d9
Merge fixes
adamhathcock Jan 26, 2026
27cf279
More LZMA fixes?
adamhathcock Jan 26, 2026
27fe2d8
more lzma porting
adamhathcock Jan 26, 2026
335db1e
fix ValueTask struct copying
adamhathcock Jan 26, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
55 changes: 55 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -179,3 +179,58 @@ SharpCompress supports multiple archive and compression formats:
3. **Stream disposal** - Always set `LeaveStreamOpen` explicitly when needed (default is to close)
4. **Tar + non-seekable stream** - Must provide file size or it will throw
6. **Format detection** - Use `ReaderFactory.Open()` for auto-detection, test with actual archive files

### Async Struct-Copy Bug in LZMA RangeCoder

When implementing async methods on mutable `struct` types (like `BitEncoder` and `BitDecoder` in the LZMA RangeCoder), be aware that the async state machine copies the struct when `await` is encountered. This means mutations to struct fields after the `await` point may not persist back to the original struct stored in arrays or fields.

**The Bug:**
```csharp
// BAD: async method on mutable struct
public async ValueTask<uint> DecodeAsync(Decoder decoder, CancellationToken cancellationToken = default)
{
var newBound = (decoder._range >> K_NUM_BIT_MODEL_TOTAL_BITS) * _prob;
if (decoder._code < newBound)
{
decoder._range = newBound;
_prob += (K_BIT_MODEL_TOTAL - _prob) >> K_NUM_MOVE_BITS; // Mutates _prob
await decoder.Normalize2Async(cancellationToken).ConfigureAwait(false); // Struct gets copied here
return 0; // Original _prob update may be lost
}
// ...
}
```

**The Fix:**
Refactor async methods on mutable structs to perform all struct mutations synchronously before any `await`, or use a helper method to separate the await from the struct mutation:

```csharp
// GOOD: struct mutations happen synchronously, await is conditional
public ValueTask<uint> DecodeAsync(Decoder decoder, CancellationToken cancellationToken = default)
{
var newBound = (decoder._range >> K_NUM_BIT_MODEL_TOTAL_BITS) * _prob;
if (decoder._code < newBound)
{
decoder._range = newBound;
_prob += (K_BIT_MODEL_TOTAL - _prob) >> K_NUM_MOVE_BITS; // All mutations complete
return DecodeAsyncHelper(decoder.Normalize2Async(cancellationToken), 0); // Await in helper
}
decoder._range -= newBound;
decoder._code -= newBound;
_prob -= (_prob) >> K_NUM_MOVE_BITS; // All mutations complete
return DecodeAsyncHelper(decoder.Normalize2Async(cancellationToken), 1); // Await in helper
}

private static async ValueTask<uint> DecodeAsyncHelper(ValueTask normalizeTask, uint result)
{
await normalizeTask.ConfigureAwait(false);
return result;
}
```

**Why This Matters:**
In LZMA, the `BitEncoder` and `BitDecoder` structs maintain adaptive probability models in their `_prob` field. When these structs are stored in arrays (e.g., `_models[m]`), the async state machine copy breaks the adaptive model, causing incorrect bit decoding and eventually `DataErrorException` exceptions.

**Related Files:**
- `src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBit.Async.cs` - Fixed
- `src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBitTree.Async.cs` - Uses readonly structs, so this pattern doesn't apply
2 changes: 1 addition & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<PackageVersion Include="System.Text.Encoding.CodePages" Version="10.0.0" />
<PackageVersion Include="System.Buffers" Version="4.6.1" />
<PackageVersion Include="System.Memory" Version="4.6.3" />
<PackageVersion Include="xunit" Version="2.9.3" />
<PackageVersion Include="xunit.v3" Version="3.2.1" />
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
<GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<GlobalPackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" />
Expand Down
103 changes: 103 additions & 0 deletions src/SharpCompress/Archives/AbstractArchive.Async.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Readers;

namespace SharpCompress.Archives;

public abstract partial class AbstractArchive<TEntry, TVolume>
where TEntry : IArchiveEntry
where TVolume : IVolume
{
#region Async Support

// Async properties
public virtual IAsyncEnumerable<TEntry> EntriesAsync => _lazyEntriesAsync;

public IAsyncEnumerable<TVolume> VolumesAsync => _lazyVolumesAsync;

protected virtual async IAsyncEnumerable<TEntry> LoadEntriesAsync(
IAsyncEnumerable<TVolume> volumes
)
{
foreach (var item in LoadEntries(await volumes.ToListAsync()))
{
yield return item;
}
}

public virtual async ValueTask DisposeAsync()
{
if (!_disposed)
{
await foreach (var v in _lazyVolumesAsync)
{
v.Dispose();
}
foreach (var v in _lazyEntriesAsync.GetLoaded().Cast<Entry>())
{
v.Close();
}
_sourceStream?.Dispose();

_disposed = true;
}
}

private async ValueTask EnsureEntriesLoadedAsync()
{
await _lazyEntriesAsync.EnsureFullyLoaded();
await _lazyVolumesAsync.EnsureFullyLoaded();
}

private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast()
{
await foreach (var entry in EntriesAsync)
{
yield return entry;
}
}

IAsyncEnumerable<IArchiveEntry> IAsyncArchive.EntriesAsync => EntriesAsyncCast();

IAsyncEnumerable<IVolume> IAsyncArchive.VolumesAsync => VolumesAsyncCast();

private async IAsyncEnumerable<IVolume> VolumesAsyncCast()
{
await foreach (var volume in _lazyVolumesAsync)
{
yield return volume;
}
}

public async ValueTask<IAsyncReader> ExtractAllEntriesAsync()
{
if (!await IsSolidAsync() && Type != ArchiveType.SevenZip)
{
throw new SharpCompressException(
"ExtractAllEntries can only be used on solid archives or 7Zip archives (which require random access)."
);
}
await EnsureEntriesLoadedAsync();
return await CreateReaderForSolidExtractionAsync();
}

public virtual ValueTask<bool> IsSolidAsync() => new(false);

public async ValueTask<bool> IsCompleteAsync()
{
await EnsureEntriesLoadedAsync();
return await EntriesAsync.AllAsync(x => x.IsComplete);
}

public async ValueTask<long> TotalSizeAsync() =>
await EntriesAsync.AggregateAsync(0L, (total, cf) => total + cf.CompressedSize);

public async ValueTask<long> TotalUncompressedSizeAsync() =>
await EntriesAsync.AggregateAsync(0L, (total, cf) => total + cf.Size);

public ValueTask<bool> IsEncryptedAsync() => new(IsEncrypted);

#endregion
}
97 changes: 5 additions & 92 deletions src/SharpCompress/Archives/AbstractArchive.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

namespace SharpCompress.Archives;

public abstract class AbstractArchive<TEntry, TVolume> : IArchive, IAsyncArchive
public abstract partial class AbstractArchive<TEntry, TVolume> : IArchive, IAsyncArchive
where TEntry : IArchiveEntry
where TVolume : IVolume
{
Expand All @@ -16,6 +16,10 @@ public abstract class AbstractArchive<TEntry, TVolume> : IArchive, IAsyncArchive
private bool _disposed;
private readonly SourceStream? _sourceStream;

// Async fields - kept in original file per refactoring rules
private readonly LazyAsyncReadOnlyCollection<TVolume> _lazyVolumesAsync;
private readonly LazyAsyncReadOnlyCollection<TEntry> _lazyEntriesAsync;

protected ReaderOptions ReaderOptions { get; }

internal AbstractArchive(ArchiveType type, SourceStream sourceStream)
Expand Down Expand Up @@ -77,16 +81,6 @@ internal AbstractArchive(ArchiveType type)
protected virtual IAsyncEnumerable<TVolume> LoadVolumesAsync(SourceStream sourceStream) =>
LoadVolumes(sourceStream).ToAsyncEnumerable();

protected virtual async IAsyncEnumerable<TEntry> LoadEntriesAsync(
IAsyncEnumerable<TVolume> volumes
)
{
foreach (var item in LoadEntries(await volumes.ToListAsync()))
{
yield return item;
}
}

IEnumerable<IArchiveEntry> IArchive.Entries => Entries.Cast<IArchiveEntry>();

IEnumerable<IVolume> IArchive.Volumes => _lazyVolumes.Cast<IVolume>();
Expand Down Expand Up @@ -156,85 +150,4 @@ public bool IsComplete
return Entries.All(x => x.IsComplete);
}
}

#region Async Support

private readonly LazyAsyncReadOnlyCollection<TVolume> _lazyVolumesAsync;
private readonly LazyAsyncReadOnlyCollection<TEntry> _lazyEntriesAsync;

public virtual async ValueTask DisposeAsync()
{
if (!_disposed)
{
await foreach (var v in _lazyVolumesAsync)
{
v.Dispose();
}
foreach (var v in _lazyEntriesAsync.GetLoaded().Cast<Entry>())
{
v.Close();
}
_sourceStream?.Dispose();

_disposed = true;
}
}

private async ValueTask EnsureEntriesLoadedAsync()
{
await _lazyEntriesAsync.EnsureFullyLoaded();
await _lazyVolumesAsync.EnsureFullyLoaded();
}

public virtual IAsyncEnumerable<TEntry> EntriesAsync => _lazyEntriesAsync;

private async IAsyncEnumerable<IArchiveEntry> EntriesAsyncCast()
{
await foreach (var entry in EntriesAsync)
{
yield return entry;
}
}

IAsyncEnumerable<IArchiveEntry> IAsyncArchive.EntriesAsync => EntriesAsyncCast();

private async IAsyncEnumerable<IVolume> VolumesAsyncCast()
{
await foreach (var volume in VolumesAsync)
{
yield return volume;
}
}

public IAsyncEnumerable<IVolume> VolumesAsync => VolumesAsyncCast();

public async ValueTask<IAsyncReader> ExtractAllEntriesAsync()
{
if (!IsSolid && Type != ArchiveType.SevenZip)
{
throw new SharpCompressException(
"ExtractAllEntries can only be used on solid archives or 7Zip archives (which require random access)."
);
}
await EnsureEntriesLoadedAsync();
return await CreateReaderForSolidExtractionAsync();
}

public virtual ValueTask<bool> IsSolidAsync() => new(false);

public async ValueTask<bool> IsCompleteAsync()
{
await EnsureEntriesLoadedAsync();
return await EntriesAsync.AllAsync(x => x.IsComplete);
}

public async ValueTask<long> TotalSizeAsync() =>
await EntriesAsync.AggregateAsync(0L, (total, cf) => total + cf.CompressedSize);

public async ValueTask<long> TotalUncompressedSizeAsync() =>
await EntriesAsync.AggregateAsync(0L, (total, cf) => total + cf.Size);

public ValueTask<bool> IsEncryptedAsync() => new(IsEncrypted);

#endregion
}
Loading
Loading