mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-05-20 23:17:00 +00:00
Merge pull request #991 from adamhathcock/async-reader-methods
Add more Async tests and complete Zip tests
This commit is contained in:
@@ -36,11 +36,10 @@ internal class StreamingZipHeaderFactory : ZipHeaderFactory
|
||||
throw new ArgumentException("Stream must be a SharpCompressStream", nameof(stream));
|
||||
}
|
||||
}
|
||||
SharpCompressStream rewindableStream = (SharpCompressStream)stream;
|
||||
var rewindableStream = (SharpCompressStream)stream;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ZipHeader? header;
|
||||
var reader = new BinaryReader(rewindableStream);
|
||||
uint headerBytes = 0;
|
||||
if (
|
||||
@@ -155,7 +154,7 @@ internal class StreamingZipHeaderFactory : ZipHeaderFactory
|
||||
}
|
||||
|
||||
_lastEntryHeader = null;
|
||||
header = ReadHeader(headerBytes, reader);
|
||||
var header = ReadHeader(headerBytes, reader);
|
||||
if (header is null)
|
||||
{
|
||||
yield break;
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
#nullable disable
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Common.Zip;
|
||||
using SharpCompress.IO;
|
||||
@@ -39,7 +39,6 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
private const int DEFAULT_BUFFER_SIZE = 8192;
|
||||
|
||||
private Stream _stream;
|
||||
private CompressionMode _mode;
|
||||
private InflaterManaged _inflater;
|
||||
private byte[] _buffer;
|
||||
|
||||
@@ -62,61 +61,23 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
throw new ArgumentException("Deflate64: input stream is not readable", nameof(stream));
|
||||
}
|
||||
|
||||
InitializeInflater(stream, ZipCompressionMethod.Deflate64);
|
||||
#if DEBUG_STREAMS
|
||||
this.DebugConstruct(typeof(Deflate64Stream));
|
||||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets up this DeflateManagedStream to be used for Inflation/Decompression
|
||||
/// </summary>
|
||||
private void InitializeInflater(
|
||||
Stream stream,
|
||||
ZipCompressionMethod method = ZipCompressionMethod.Deflate
|
||||
)
|
||||
{
|
||||
Debug.Assert(stream != null);
|
||||
Debug.Assert(
|
||||
method == ZipCompressionMethod.Deflate || method == ZipCompressionMethod.Deflate64
|
||||
);
|
||||
if (!stream.CanRead)
|
||||
{
|
||||
throw new ArgumentException("Deflate64: input stream is not readable", nameof(stream));
|
||||
}
|
||||
|
||||
_inflater = new InflaterManaged(method == ZipCompressionMethod.Deflate64);
|
||||
_inflater = new InflaterManaged(true);
|
||||
|
||||
_stream = stream;
|
||||
_mode = CompressionMode.Decompress;
|
||||
_buffer = new byte[DEFAULT_BUFFER_SIZE];
|
||||
#if DEBUG_STREAMS
|
||||
this.DebugConstruct(typeof(Deflate64Stream));
|
||||
#endif
|
||||
}
|
||||
|
||||
public override bool CanRead
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_stream is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
public override bool CanRead => _stream.CanRead;
|
||||
|
||||
return (_mode == CompressionMode.Decompress && _stream.CanRead);
|
||||
}
|
||||
}
|
||||
|
||||
public override bool CanWrite
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_stream is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return (_mode == CompressionMode.Compress && _stream.CanWrite);
|
||||
}
|
||||
}
|
||||
public override bool CanWrite => false;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
@@ -138,7 +99,6 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
|
||||
public override int Read(byte[] array, int offset, int count)
|
||||
{
|
||||
EnsureDecompressionMode();
|
||||
ValidateParameters(array, offset, count);
|
||||
EnsureNotDisposed();
|
||||
|
||||
@@ -185,6 +145,106 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
return count - remainingCount;
|
||||
}
|
||||
|
||||
public override async Task<int> ReadAsync(
|
||||
byte[] array,
|
||||
int offset,
|
||||
int count,
|
||||
CancellationToken cancellationToken
|
||||
)
|
||||
{
|
||||
ValidateParameters(array, offset, count);
|
||||
EnsureNotDisposed();
|
||||
|
||||
int bytesRead;
|
||||
var currentOffset = offset;
|
||||
var remainingCount = count;
|
||||
|
||||
while (true)
|
||||
{
|
||||
bytesRead = _inflater.Inflate(array, currentOffset, remainingCount);
|
||||
currentOffset += bytesRead;
|
||||
remainingCount -= bytesRead;
|
||||
|
||||
if (remainingCount == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (_inflater.Finished())
|
||||
{
|
||||
// if we finished decompressing, we can't have anything left in the outputwindow.
|
||||
Debug.Assert(
|
||||
_inflater.AvailableOutput == 0,
|
||||
"We should have copied all stuff out!"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
var bytes = await _stream
|
||||
.ReadAsync(_buffer, 0, _buffer.Length, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (bytes <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
else if (bytes > _buffer.Length)
|
||||
{
|
||||
// The stream is either malicious or poorly implemented and returned a number of
|
||||
// bytes larger than the buffer supplied to it.
|
||||
throw new InvalidFormatException("Deflate64: invalid data");
|
||||
}
|
||||
|
||||
_inflater.SetInput(_buffer, 0, bytes);
|
||||
}
|
||||
|
||||
return count - remainingCount;
|
||||
}
|
||||
|
||||
#if !NETFRAMEWORK && !NETSTANDARD2_0
|
||||
public override async ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
EnsureNotDisposed();
|
||||
|
||||
// InflaterManaged doesn't have a Span-based Inflate method, so we need to work with arrays
|
||||
// For large buffers, we could rent from ArrayPool, but for simplicity we'll use the buffer's array if available
|
||||
if (
|
||||
System.Runtime.InteropServices.MemoryMarshal.TryGetArray<byte>(
|
||||
buffer,
|
||||
out var arraySegment
|
||||
)
|
||||
)
|
||||
{
|
||||
// Fast path: the Memory<byte> is backed by an array
|
||||
return await ReadAsync(
|
||||
arraySegment.Array!,
|
||||
arraySegment.Offset,
|
||||
arraySegment.Count,
|
||||
cancellationToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Slow path: rent a temporary array
|
||||
var tempBuffer = System.Buffers.ArrayPool<byte>.Shared.Rent(buffer.Length);
|
||||
try
|
||||
{
|
||||
var bytesRead = await ReadAsync(tempBuffer, 0, buffer.Length, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
tempBuffer.AsMemory(0, bytesRead).CopyTo(buffer);
|
||||
return bytesRead;
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(tempBuffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
private void ValidateParameters(byte[] array, int offset, int count)
|
||||
{
|
||||
if (array is null)
|
||||
@@ -220,26 +280,6 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
private static void ThrowStreamClosedException() =>
|
||||
throw new ObjectDisposedException(null, "Deflate64: stream has been disposed");
|
||||
|
||||
private void EnsureDecompressionMode()
|
||||
{
|
||||
if (_mode != CompressionMode.Decompress)
|
||||
{
|
||||
ThrowCannotReadFromDeflateManagedStreamException();
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
private static void ThrowCannotReadFromDeflateManagedStreamException() =>
|
||||
throw new InvalidOperationException("Deflate64: cannot read from this stream");
|
||||
|
||||
private void EnsureCompressionMode()
|
||||
{
|
||||
if (_mode != CompressionMode.Compress)
|
||||
{
|
||||
ThrowCannotWriteToDeflateManagedStreamException();
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
private static void ThrowCannotWriteToDeflateManagedStreamException() =>
|
||||
throw new InvalidOperationException("Deflate64: cannot write to this stream");
|
||||
@@ -281,20 +321,17 @@ public sealed class Deflate64Stream : Stream, IStreamStack
|
||||
#endif
|
||||
if (disposing)
|
||||
{
|
||||
_stream?.Dispose();
|
||||
_stream.Dispose();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_stream = null;
|
||||
|
||||
try
|
||||
{
|
||||
_inflater?.Dispose();
|
||||
_inflater.Dispose();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_inflater = null;
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace SharpCompress.IO;
|
||||
|
||||
@@ -93,6 +95,47 @@ internal class ReadOnlySubStream : SharpCompressStream, IStreamStack
|
||||
}
|
||||
#endif
|
||||
|
||||
public override async Task<int> ReadAsync(
|
||||
byte[] buffer,
|
||||
int offset,
|
||||
int count,
|
||||
CancellationToken cancellationToken
|
||||
)
|
||||
{
|
||||
if (BytesLeftToRead < count)
|
||||
{
|
||||
count = (int)BytesLeftToRead;
|
||||
}
|
||||
var read = await Stream
|
||||
.ReadAsync(buffer, offset, count, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (read > 0)
|
||||
{
|
||||
BytesLeftToRead -= read;
|
||||
_position += read;
|
||||
}
|
||||
return read;
|
||||
}
|
||||
|
||||
#if !NETFRAMEWORK && !NETSTANDARD2_0
|
||||
public override async ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
var sliceLen = BytesLeftToRead < buffer.Length ? BytesLeftToRead : buffer.Length;
|
||||
var read = await Stream
|
||||
.ReadAsync(buffer.Slice(0, (int)sliceLen), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (read > 0)
|
||||
{
|
||||
BytesLeftToRead -= read;
|
||||
_position += read;
|
||||
}
|
||||
return read;
|
||||
}
|
||||
#endif
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
|
||||
@@ -96,6 +96,33 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader, IReaderExtracti
|
||||
return false;
|
||||
}
|
||||
|
||||
public async Task<bool> MoveToNextEntryAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_completed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (Cancelled)
|
||||
{
|
||||
throw new ReaderCancelledException("Reader has been cancelled.");
|
||||
}
|
||||
if (_entriesForCurrentReadStream is null)
|
||||
{
|
||||
return LoadStreamForReading(RequestInitialStream());
|
||||
}
|
||||
if (!_wroteCurrentEntry)
|
||||
{
|
||||
await SkipEntryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
_wroteCurrentEntry = false;
|
||||
if (NextEntryForCurrentStream())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_completed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
protected bool LoadStreamForReading(Stream stream)
|
||||
{
|
||||
_entriesForCurrentReadStream?.Dispose();
|
||||
@@ -129,6 +156,14 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader, IReaderExtracti
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SkipEntryAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Entry.IsDirectory)
|
||||
{
|
||||
await SkipAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void Skip()
|
||||
{
|
||||
var part = Entry.Parts.First();
|
||||
@@ -151,6 +186,33 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader, IReaderExtracti
|
||||
s.SkipEntry();
|
||||
}
|
||||
|
||||
private async Task SkipAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var part = Entry.Parts.First();
|
||||
|
||||
if (!Entry.IsSplitAfter && !Entry.IsSolid && Entry.CompressedSize > 0)
|
||||
{
|
||||
//not solid and has a known compressed size then we can skip raw bytes.
|
||||
var rawStream = part.GetRawStream();
|
||||
|
||||
if (rawStream != null)
|
||||
{
|
||||
var bytesToAdvance = Entry.CompressedSize;
|
||||
await rawStream.SkipAsync(bytesToAdvance, cancellationToken).ConfigureAwait(false);
|
||||
part.Skipped = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
//don't know the size so we have to try to decompress to skip
|
||||
#if NETFRAMEWORK || NETSTANDARD2_0
|
||||
using var s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await s.SkipEntryAsync(cancellationToken).ConfigureAwait(false);
|
||||
#else
|
||||
await using var s = await OpenEntryStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await s.SkipEntryAsync(cancellationToken).ConfigureAwait(false);
|
||||
#endif
|
||||
}
|
||||
|
||||
public void WriteEntryTo(Stream writableStream)
|
||||
{
|
||||
if (_wroteCurrentEntry)
|
||||
@@ -232,6 +294,19 @@ public abstract class AbstractReader<TEntry, TVolume> : IReader, IReaderExtracti
|
||||
return stream;
|
||||
}
|
||||
|
||||
public Task<EntryStream> OpenEntryStreamAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_wroteCurrentEntry)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
"WriteEntryToAsync or OpenEntryStreamAsync can only be called once."
|
||||
);
|
||||
}
|
||||
var stream = GetEntryStream();
|
||||
_wroteCurrentEntry = true;
|
||||
return Task.FromResult(stream);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retains a reference to the entry stream, so we can check whether it completed later.
|
||||
/// </summary>
|
||||
|
||||
@@ -39,9 +39,23 @@ public interface IReader : IDisposable
|
||||
/// <returns></returns>
|
||||
bool MoveToNextEntry();
|
||||
|
||||
/// <summary>
|
||||
/// Moves to the next entry asynchronously by reading more data from the underlying stream. This skips if data has not been read.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken"></param>
|
||||
/// <returns></returns>
|
||||
Task<bool> MoveToNextEntryAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Opens the current entry as a stream that will decompress as it is read.
|
||||
/// Read the entire stream or use SkipEntry on EntryStream.
|
||||
/// </summary>
|
||||
EntryStream OpenEntryStream();
|
||||
|
||||
/// <summary>
|
||||
/// Opens the current entry asynchronously as a stream that will decompress as it is read.
|
||||
/// Read the entire stream or use SkipEntry on EntryStream.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken"></param>
|
||||
Task<EntryStream> OpenEntryStreamAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -335,9 +335,9 @@
|
||||
"net8.0": {
|
||||
"Microsoft.NET.ILLink.Tasks": {
|
||||
"type": "Direct",
|
||||
"requested": "[8.0.0, )",
|
||||
"resolved": "8.0.0",
|
||||
"contentHash": "B3etT5XQ2nlWkZGO2m/ytDYrOmSsQG1XNBaM6ZYlX5Ch/tDrMFadr0/mK6gjZwaQc55g+5+WZMw4Cz3m8VEF7g=="
|
||||
"requested": "[8.0.17, )",
|
||||
"resolved": "8.0.17",
|
||||
"contentHash": "x5/y4l8AtshpBOrCZdlE4txw8K3e3s9meBFeZeR3l8hbbku2V7kK6ojhXvrbjg1rk3G+JqL1BI26gtgc1ZrdUw=="
|
||||
},
|
||||
"Microsoft.SourceLink.GitHub": {
|
||||
"type": "Direct",
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.IO;
|
||||
using SharpCompress.Readers;
|
||||
|
||||
@@ -57,7 +59,7 @@ public class ForwardOnlyStream : SharpCompressStream, IStreamStack
|
||||
public override bool CanRead => true;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override bool CanWrite => true;
|
||||
|
||||
public override void Flush() { }
|
||||
|
||||
@@ -72,10 +74,41 @@ public class ForwardOnlyStream : SharpCompressStream, IStreamStack
|
||||
public override int Read(byte[] buffer, int offset, int count) =>
|
||||
stream.Read(buffer, offset, count);
|
||||
|
||||
public override Task<int> ReadAsync(
|
||||
byte[] buffer,
|
||||
int offset,
|
||||
int count,
|
||||
CancellationToken cancellationToken
|
||||
) => stream.ReadAsync(buffer, offset, count, cancellationToken);
|
||||
|
||||
#if !NETFRAMEWORK && !NETSTANDARD2_0
|
||||
public override ValueTask<int> ReadAsync(
|
||||
Memory<byte> buffer,
|
||||
CancellationToken cancellationToken = default
|
||||
) => stream.ReadAsync(buffer, cancellationToken);
|
||||
#endif
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count) =>
|
||||
throw new NotSupportedException();
|
||||
stream.Write(buffer, offset, count);
|
||||
|
||||
public override Task WriteAsync(
|
||||
byte[] buffer,
|
||||
int offset,
|
||||
int count,
|
||||
CancellationToken cancellationToken
|
||||
) => stream.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
|
||||
#if !NETFRAMEWORK && !NETSTANDARD2_0
|
||||
public override ValueTask WriteAsync(
|
||||
ReadOnlyMemory<byte> buffer,
|
||||
CancellationToken cancellationToken = default
|
||||
) => stream.WriteAsync(buffer, cancellationToken);
|
||||
#endif
|
||||
|
||||
public override Task FlushAsync(CancellationToken cancellationToken) =>
|
||||
stream.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ public abstract class ReaderTests : TestBase
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
while (reader.MoveToNextEntry())
|
||||
while (await reader.MoveToNextEntryAsync(cancellationToken))
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
|
||||
242
tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs
Normal file
242
tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs
Normal file
@@ -0,0 +1,242 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Common.Zip;
|
||||
using SharpCompress.Compressors.Deflate;
|
||||
using SharpCompress.Readers;
|
||||
using SharpCompress.Readers.Zip;
|
||||
using SharpCompress.Test.Mocks;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class Zip64AsyncTests : WriterTests
|
||||
{
|
||||
public Zip64AsyncTests()
|
||||
: base(ArchiveType.Zip) { }
|
||||
|
||||
// 4GiB + 1
|
||||
private const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1;
|
||||
|
||||
//[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Single_Large_File_Async() =>
|
||||
await RunSingleTestAsync(1, FOUR_GB_LIMIT, setZip64: true, forwardOnly: false);
|
||||
|
||||
//[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Two_Large_Files_Async() =>
|
||||
await RunSingleTestAsync(2, FOUR_GB_LIMIT, setZip64: true, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Two_Small_files_Async() =>
|
||||
// Multiple files, does not require zip64
|
||||
await RunSingleTestAsync(2, FOUR_GB_LIMIT / 2, setZip64: false, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Two_Small_files_stream_Async() =>
|
||||
await RunSingleTestAsync(2, FOUR_GB_LIMIT / 2, setZip64: false, forwardOnly: true);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Two_Small_Files_Zip64_Async() =>
|
||||
// Multiple files, use zip64 even though it is not required
|
||||
await RunSingleTestAsync(2, FOUR_GB_LIMIT / 2, setZip64: true, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public async Task Zip64_Single_Large_File_Fail_Async()
|
||||
{
|
||||
try
|
||||
{
|
||||
// One single file, should fail
|
||||
await RunSingleTestAsync(1, FOUR_GB_LIMIT, setZip64: false, forwardOnly: false);
|
||||
throw new InvalidOperationException("Test did not fail?");
|
||||
}
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("zip64", "true")]
|
||||
public async Task Zip64_Single_Large_File_Zip64_Streaming_Fail_Async()
|
||||
{
|
||||
try
|
||||
{
|
||||
// One single file, should fail (fast) with zip64
|
||||
await RunSingleTestAsync(1, FOUR_GB_LIMIT, setZip64: true, forwardOnly: true);
|
||||
throw new InvalidOperationException("Test did not fail?");
|
||||
}
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("zip64", "true")]
|
||||
public async Task Zip64_Single_Large_File_Streaming_Fail_Async()
|
||||
{
|
||||
try
|
||||
{
|
||||
// One single file, should fail once the write discovers the problem
|
||||
await RunSingleTestAsync(1, FOUR_GB_LIMIT, setZip64: false, forwardOnly: true);
|
||||
throw new InvalidOperationException("Test did not fail?");
|
||||
}
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
public async Task RunSingleTestAsync(
|
||||
long files,
|
||||
long filesize,
|
||||
bool setZip64,
|
||||
bool forwardOnly,
|
||||
long writeChunkSize = 1024 * 1024,
|
||||
string filename = "zip64-test-async.zip"
|
||||
)
|
||||
{
|
||||
filename = Path.Combine(SCRATCH2_FILES_PATH, filename);
|
||||
|
||||
try
|
||||
{
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
File.Delete(filename);
|
||||
}
|
||||
|
||||
if (!File.Exists(filename))
|
||||
{
|
||||
await CreateZipArchiveAsync(
|
||||
filename,
|
||||
files,
|
||||
filesize,
|
||||
writeChunkSize,
|
||||
setZip64,
|
||||
forwardOnly
|
||||
);
|
||||
}
|
||||
|
||||
var resForward = await ReadForwardOnlyAsync(filename);
|
||||
if (resForward.Item1 != files)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Incorrect number of items reported: {resForward.Item1}, should have been {files}"
|
||||
);
|
||||
}
|
||||
|
||||
if (resForward.Item2 != files * filesize)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Incorrect combined size reported: {resForward.Item2}, should have been {files * filesize}"
|
||||
);
|
||||
}
|
||||
|
||||
var resArchive = ReadArchive(filename);
|
||||
if (resArchive.Item1 != files)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Incorrect number of items reported: {resArchive.Item1}, should have been {files}"
|
||||
);
|
||||
}
|
||||
|
||||
if (resArchive.Item2 != files * filesize)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Incorrect number of items reported: {resArchive.Item2}, should have been {files * filesize}"
|
||||
);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(filename))
|
||||
{
|
||||
File.Delete(filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CreateZipArchiveAsync(
|
||||
string filename,
|
||||
long files,
|
||||
long filesize,
|
||||
long chunksize,
|
||||
bool setZip64,
|
||||
bool forwardOnly
|
||||
)
|
||||
{
|
||||
var data = new byte[chunksize];
|
||||
|
||||
// Use deflate for speed
|
||||
var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = setZip64 };
|
||||
|
||||
// Use no compression to ensure we hit the limits (actually inflates a bit, but seems better than using method==Store)
|
||||
var eo = new ZipWriterEntryOptions { DeflateCompressionLevel = CompressionLevel.None };
|
||||
|
||||
using var zip = File.OpenWrite(filename);
|
||||
using var st = forwardOnly ? (Stream)new ForwardOnlyStream(zip) : zip;
|
||||
using var zipWriter = (ZipWriter)WriterFactory.Open(st, ArchiveType.Zip, opts);
|
||||
for (var i = 0; i < files; i++)
|
||||
{
|
||||
using var str = zipWriter.WriteToStream(i.ToString(), eo);
|
||||
var left = filesize;
|
||||
while (left > 0)
|
||||
{
|
||||
var b = (int)Math.Min(left, data.Length);
|
||||
// Use synchronous Write to match the sync version and avoid ForwardOnlyStream issues
|
||||
await str.WriteAsync(data, 0, b);
|
||||
left -= b;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Tuple<long, long>> ReadForwardOnlyAsync(string filename)
|
||||
{
|
||||
long count = 0;
|
||||
long size = 0;
|
||||
ZipEntry? prev = null;
|
||||
using (var fs = File.OpenRead(filename))
|
||||
using (var rd = ZipReader.Open(fs, new ReaderOptions { LookForHeader = false }))
|
||||
{
|
||||
while (await rd.MoveToNextEntryAsync())
|
||||
{
|
||||
#if NETFRAMEWORK || NETSTANDARD2_0
|
||||
using (var entryStream = await rd.OpenEntryStreamAsync())
|
||||
{
|
||||
await entryStream.SkipEntryAsync();
|
||||
}
|
||||
#else
|
||||
await using (var entryStream = await rd.OpenEntryStreamAsync())
|
||||
{
|
||||
await entryStream.SkipEntryAsync();
|
||||
}
|
||||
#endif
|
||||
count++;
|
||||
if (prev != null)
|
||||
{
|
||||
size += prev.Size;
|
||||
}
|
||||
|
||||
prev = rd.Entry;
|
||||
}
|
||||
}
|
||||
|
||||
if (prev != null)
|
||||
{
|
||||
size += prev.Size;
|
||||
}
|
||||
|
||||
return new Tuple<long, long>(count, size);
|
||||
}
|
||||
|
||||
public Tuple<long, long> ReadArchive(string filename)
|
||||
{
|
||||
using var archive = ArchiveFactory.Open(filename);
|
||||
return new Tuple<long, long>(
|
||||
archive.Entries.Count(),
|
||||
archive.Entries.Select(x => x.Size).Sum()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -22,31 +22,37 @@ public class Zip64Tests : WriterTests
|
||||
// 4GiB + 1
|
||||
private const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1;
|
||||
|
||||
//[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Single_Large_File() =>
|
||||
// One single file, requires zip64
|
||||
RunSingleTest(1, FOUR_GB_LIMIT, setZip64: true, forwardOnly: false);
|
||||
|
||||
//[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Two_Large_Files() =>
|
||||
// One single file, requires zip64
|
||||
RunSingleTest(2, FOUR_GB_LIMIT, setZip64: true, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Two_Small_files() =>
|
||||
// Multiple files, does not require zip64
|
||||
RunSingleTest(2, FOUR_GB_LIMIT / 2, setZip64: false, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Two_Small_files_stream() =>
|
||||
// Multiple files, does not require zip64, and works with streams
|
||||
RunSingleTest(2, FOUR_GB_LIMIT / 2, setZip64: false, forwardOnly: true);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Two_Small_Files_Zip64() =>
|
||||
// Multiple files, use zip64 even though it is not required
|
||||
RunSingleTest(2, FOUR_GB_LIMIT / 2, setZip64: true, forwardOnly: false);
|
||||
|
||||
[Fact]
|
||||
[Trait("format", "zip64")]
|
||||
public void Zip64_Single_Large_File_Fail()
|
||||
{
|
||||
@@ -59,6 +65,7 @@ public class Zip64Tests : WriterTests
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("zip64", "true")]
|
||||
public void Zip64_Single_Large_File_Zip64_Streaming_Fail()
|
||||
{
|
||||
@@ -71,6 +78,7 @@ public class Zip64Tests : WriterTests
|
||||
catch (NotSupportedException) { }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("zip64", "true")]
|
||||
public void Zip64_Single_Large_File_Streaming_Fail()
|
||||
{
|
||||
|
||||
196
tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs
Normal file
196
tests/SharpCompress.Test/Zip/ZipArchiveAsyncTests.cs
Normal file
@@ -0,0 +1,196 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Archives.Zip;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class ZipArchiveAsyncTests : ArchiveTests
|
||||
{
|
||||
public ZipArchiveAsyncTests() => UseExtensionInsteadOfNameToVerify = true;
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_ZipX_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.zipx");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_Streamed_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.bzip2.dd.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.bzip2.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Streamed2_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.deflate.dd-.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Streamed_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.deflate.dd.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.deflate.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate64_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.deflate64.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_Streamed_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.lzma.dd.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.lzma.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_PPMd_Streamed_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.ppmd.dd.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_PPMd_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.ppmd.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_None_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.none.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Zip64_ArchiveStreamRead_Async() =>
|
||||
await ArchiveStreamReadAsync("Zip.zip64.zip");
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Shrink_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.shrink.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Implode_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.implode.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reduce1_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.reduce1.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reduce2_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.reduce2.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reduce3_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.reduce3.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reduce4_ArchiveStreamRead_Async()
|
||||
{
|
||||
UseExtensionInsteadOfNameToVerify = true;
|
||||
UseCaseInsensitiveToVerify = true;
|
||||
await ArchiveStreamReadAsync("Zip.reduce4.zip");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Random_Write_Remove_Async()
|
||||
{
|
||||
var scratchPath = Path.Combine(SCRATCH_FILES_PATH, "Zip.deflate.mod.zip");
|
||||
var unmodified = Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.noEmptyDirs.zip");
|
||||
var modified = Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.mod.zip");
|
||||
|
||||
using (var archive = ZipArchive.Open(unmodified))
|
||||
{
|
||||
var entry = archive.Entries.Single(x =>
|
||||
x.Key.NotNull().EndsWith("jpg", StringComparison.OrdinalIgnoreCase)
|
||||
);
|
||||
archive.RemoveEntry(entry);
|
||||
|
||||
WriterOptions writerOptions = new ZipWriterOptions(CompressionType.Deflate);
|
||||
writerOptions.ArchiveEncoding.Default = Encoding.GetEncoding(866);
|
||||
|
||||
await archive.SaveToAsync(scratchPath, writerOptions);
|
||||
}
|
||||
CompareArchivesByPath(modified, scratchPath, Encoding.GetEncoding(866));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Random_Write_Add_Async()
|
||||
{
|
||||
var jpg = Path.Combine(ORIGINAL_FILES_PATH, "jpg", "test.jpg");
|
||||
var scratchPath = Path.Combine(SCRATCH_FILES_PATH, "Zip.deflate.mod.zip");
|
||||
var unmodified = Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.mod.zip");
|
||||
var modified = Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.noEmptyDirs.zip");
|
||||
|
||||
using (var archive = ZipArchive.Open(unmodified))
|
||||
{
|
||||
archive.AddEntry("jpg\\test.jpg", jpg);
|
||||
|
||||
WriterOptions writerOptions = new ZipWriterOptions(CompressionType.Deflate);
|
||||
writerOptions.ArchiveEncoding.Default = Encoding.GetEncoding(866);
|
||||
|
||||
await archive.SaveToAsync(scratchPath, writerOptions);
|
||||
}
|
||||
CompareArchivesByPath(modified, scratchPath, Encoding.GetEncoding(866));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Create_New_Async()
|
||||
{
|
||||
var scratchPath = Path.Combine(SCRATCH_FILES_PATH, "Zip.deflate.noEmptyDirs.zip");
|
||||
var unmodified = Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.noEmptyDirs.zip");
|
||||
|
||||
using (var archive = ZipArchive.Create())
|
||||
{
|
||||
archive.DeflateCompressionLevel = Compressors.Deflate.CompressionLevel.BestSpeed;
|
||||
archive.AddAllFromDirectory(ORIGINAL_FILES_PATH);
|
||||
|
||||
WriterOptions writerOptions = new ZipWriterOptions(CompressionType.Deflate);
|
||||
writerOptions.ArchiveEncoding.Default = Encoding.UTF8;
|
||||
|
||||
await archive.SaveToAsync(scratchPath, writerOptions);
|
||||
}
|
||||
CompareArchivesByPath(unmodified, scratchPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Entry_Stream_Async()
|
||||
{
|
||||
using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.zip")))
|
||||
using (var archive = ZipArchive.Open(stream))
|
||||
{
|
||||
foreach (var entry in archive.Entries.Where(entry => !entry.IsDirectory))
|
||||
{
|
||||
await entry.WriteToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
VerifyFiles();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Archives.Zip;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Deflate;
|
||||
using SharpCompress.Compressors.Xz;
|
||||
using SharpCompress.Crypto;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class ZipTypesLevelsWithCrcRatioAsyncTests : ArchiveTests
|
||||
{
|
||||
public ZipTypesLevelsWithCrcRatioAsyncTests() => UseExtensionInsteadOfNameToVerify = true;
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 1, 0.11f)] // was 0.8f, actual 0.104
|
||||
[InlineData(CompressionType.Deflate, 3, 1, 0.08f)] // was 0.8f, actual 0.078
|
||||
[InlineData(CompressionType.Deflate, 6, 1, 0.05f)] // was 0.8f, actual ~0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 1, 0.04f)] // was 0.7f, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 1, 1, 0.025f)] // was 0.8f, actual 0.023
|
||||
[InlineData(CompressionType.ZStandard, 3, 1, 0.015f)] // was 0.7f, actual 0.013
|
||||
[InlineData(CompressionType.ZStandard, 9, 1, 0.006f)] // was 0.7f, actual 0.005
|
||||
[InlineData(CompressionType.ZStandard, 22, 1, 0.005f)] // was 0.7f, actual 0.004
|
||||
[InlineData(CompressionType.BZip2, 0, 1, 0.035f)] // was 0.8f, actual 0.033
|
||||
[InlineData(CompressionType.LZMA, 0, 1, 0.005f)] // was 0.8f, actual 0.004
|
||||
[InlineData(CompressionType.None, 0, 1, 1.001f)] // was 1.1f, actual 1.000
|
||||
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8f, actual 0.042
|
||||
[InlineData(CompressionType.ZStandard, 3, 2, 0.012f)] // was 0.7f, actual 0.010
|
||||
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8f, actual 0.032
|
||||
[InlineData(CompressionType.Deflate, 9, 3, 0.04f)] // was 0.7f, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 9, 3, 0.003f)] // was 0.7f, actual 0.002
|
||||
public async Task Zip_Create_Archive_With_3_Files_Crc32_Test_Async(
|
||||
CompressionType compressionType,
|
||||
int compressionLevel,
|
||||
int sizeMb,
|
||||
float expectedRatio
|
||||
)
|
||||
{
|
||||
const int OneMiB = 1024 * 1024;
|
||||
var baseSize = sizeMb * OneMiB;
|
||||
|
||||
// Generate test content for files with sizes based on the sizeMb parameter
|
||||
var file1Data = TestPseudoTextStream.Create(baseSize);
|
||||
var file2Data = TestPseudoTextStream.Create(baseSize * 2);
|
||||
var file3Data = TestPseudoTextStream.Create(baseSize * 3);
|
||||
|
||||
var expectedFiles = new Dictionary<string, (byte[] data, uint crc)>
|
||||
{
|
||||
[$"file1_{sizeMb}MiB.txt"] = (file1Data, CalculateCrc32(file1Data)),
|
||||
[$"data/file2_{sizeMb * 2}MiB.txt"] = (file2Data, CalculateCrc32(file2Data)),
|
||||
[$"deep/nested/file3_{sizeMb * 3}MiB.txt"] = (file3Data, CalculateCrc32(file3Data)),
|
||||
};
|
||||
|
||||
// Create zip archive in memory
|
||||
using var zipStream = new MemoryStream();
|
||||
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
|
||||
{
|
||||
await writer.WriteAsync($"file1_{sizeMb}MiB.txt", new MemoryStream(file1Data));
|
||||
await writer.WriteAsync($"data/file2_{sizeMb * 2}MiB.txt", new MemoryStream(file2Data));
|
||||
await writer.WriteAsync(
|
||||
$"deep/nested/file3_{sizeMb * 3}MiB.txt",
|
||||
new MemoryStream(file3Data)
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var originalSize = file1Data.Length + file2Data.Length + file3Data.Length;
|
||||
var actualRatio = (double)zipStream.Length / originalSize;
|
||||
//Debug.WriteLine($"Zip_Create_Archive_With_3_Files_Crc32_Test_Async: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
// Verify compression occurred (except for None compression type)
|
||||
if (compressionType != CompressionType.None)
|
||||
{
|
||||
Assert.True(
|
||||
zipStream.Length < originalSize,
|
||||
$"Compression failed: compressed={zipStream.Length}, original={originalSize}"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify compression ratio
|
||||
VerifyCompressionRatio(
|
||||
originalSize,
|
||||
zipStream.Length,
|
||||
expectedRatio,
|
||||
$"{compressionType} level {compressionLevel}"
|
||||
);
|
||||
|
||||
// Verify archive content and CRC32
|
||||
await VerifyArchiveContentAsync(zipStream, expectedFiles);
|
||||
|
||||
// Verify compression type is correctly set
|
||||
VerifyCompressionType(zipStream, compressionType);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 4, 0.11f)] // was 0.8, actual 0.105
|
||||
[InlineData(CompressionType.Deflate, 3, 4, 0.08f)] // was 0.8, actual 0.077
|
||||
[InlineData(CompressionType.Deflate, 6, 4, 0.045f)] // was 0.8, actual 0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 4, 0.04f)] // was 0.8, actual 0.037
|
||||
[InlineData(CompressionType.ZStandard, 1, 4, 0.025f)] // was 0.8, actual 0.022
|
||||
[InlineData(CompressionType.ZStandard, 3, 4, 0.012f)] // was 0.8, actual 0.010
|
||||
[InlineData(CompressionType.ZStandard, 9, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
[InlineData(CompressionType.ZStandard, 22, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
[InlineData(CompressionType.BZip2, 0, 4, 0.035f)] // was 0.8, actual 0.032
|
||||
[InlineData(CompressionType.LZMA, 0, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
public async Task Zip_WriterFactory_Crc32_Test_Async(
|
||||
CompressionType compressionType,
|
||||
int compressionLevel,
|
||||
int sizeMb,
|
||||
float expectedRatio
|
||||
)
|
||||
{
|
||||
var fileSize = sizeMb * 1024 * 1024;
|
||||
|
||||
var testData = TestPseudoTextStream.Create(fileSize);
|
||||
var expectedCrc = CalculateCrc32(testData);
|
||||
|
||||
// Create archive with specified compression level
|
||||
using var zipStream = new MemoryStream();
|
||||
var writerOptions = new ZipWriterOptions(compressionType)
|
||||
{
|
||||
CompressionLevel = compressionLevel,
|
||||
};
|
||||
|
||||
using (var writer = WriterFactory.Open(zipStream, ArchiveType.Zip, writerOptions))
|
||||
{
|
||||
await writer.WriteAsync(
|
||||
$"{compressionType}_level_{compressionLevel}_{sizeMb}MiB.txt",
|
||||
new MemoryStream(testData)
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var actualRatio = (double)zipStream.Length / testData.Length;
|
||||
//Debug.WriteLine($"Zip_WriterFactory_Crc32_Test_Async: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
VerifyCompressionRatio(
|
||||
testData.Length,
|
||||
zipStream.Length,
|
||||
expectedRatio,
|
||||
$"{compressionType} level {compressionLevel}"
|
||||
);
|
||||
|
||||
// Verify the archive
|
||||
zipStream.Position = 0;
|
||||
using var archive = ZipArchive.Open(zipStream);
|
||||
|
||||
var entry = archive.Entries.Single(e => !e.IsDirectory);
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
await entryStream.CopyToAsync(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(compressionType, entry.CompressionType);
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(testData.Length, extractedData.Length);
|
||||
Assert.Equal(testData, extractedData);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 2, 0.11f)] // was 0.8, actual 0.104
|
||||
[InlineData(CompressionType.Deflate, 3, 2, 0.08f)] // was 0.8, actual 0.077
|
||||
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8, actual 0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 2, 0.04f)] // was 0.7, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 1, 2, 0.025f)] // was 0.8, actual 0.023
|
||||
[InlineData(CompressionType.ZStandard, 3, 2, 0.015f)] // was 0.7, actual 0.012
|
||||
[InlineData(CompressionType.ZStandard, 9, 2, 0.006f)] // was 0.7, actual 0.005
|
||||
[InlineData(CompressionType.ZStandard, 22, 2, 0.005f)] // was 0.7, actual 0.004
|
||||
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8, actual 0.032
|
||||
[InlineData(CompressionType.LZMA, 0, 2, 0.005f)] // was 0.8, actual 0.004
|
||||
public async Task Zip_ZipArchiveOpen_Crc32_Test_Async(
|
||||
CompressionType compressionType,
|
||||
int compressionLevel,
|
||||
int sizeMb,
|
||||
float expectedRatio
|
||||
)
|
||||
{
|
||||
var fileSize = sizeMb * 1024 * 1024;
|
||||
|
||||
var testData = TestPseudoTextStream.Create(fileSize);
|
||||
var expectedCrc = CalculateCrc32(testData);
|
||||
|
||||
// Create archive with specified compression and level
|
||||
using var zipStream = new MemoryStream();
|
||||
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
|
||||
{
|
||||
await writer.WriteAsync(
|
||||
$"{compressionType}_{compressionLevel}_{sizeMb}MiB.txt",
|
||||
new MemoryStream(testData)
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var actualRatio = (double)zipStream.Length / testData.Length;
|
||||
//Debug.WriteLine($"Zip_ZipArchiveOpen_Crc32_Test_Async: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
// Verify the archive
|
||||
zipStream.Position = 0;
|
||||
using var archive = ZipArchive.Open(zipStream);
|
||||
|
||||
var entry = archive.Entries.Single(e => !e.IsDirectory);
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
await entryStream.CopyToAsync(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(compressionType, entry.CompressionType);
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(testData.Length, extractedData.Length);
|
||||
|
||||
// For smaller files, verify full content; for larger, spot check
|
||||
if (testData.Length <= sizeMb * 2)
|
||||
{
|
||||
Assert.Equal(testData, extractedData);
|
||||
}
|
||||
else
|
||||
{
|
||||
VerifyDataSpotCheck(testData, extractedData);
|
||||
}
|
||||
|
||||
VerifyCompressionRatio(
|
||||
testData.Length,
|
||||
zipStream.Length,
|
||||
expectedRatio,
|
||||
$"{compressionType} Level {compressionLevel}"
|
||||
);
|
||||
}
|
||||
|
||||
// Helper method for async archive content verification
|
||||
private async Task VerifyArchiveContentAsync(
|
||||
MemoryStream zipStream,
|
||||
Dictionary<string, (byte[] data, uint crc)> expectedFiles
|
||||
)
|
||||
{
|
||||
zipStream.Position = 0;
|
||||
using var archive = ZipArchive.Open(zipStream);
|
||||
|
||||
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
|
||||
{
|
||||
Assert.True(
|
||||
expectedFiles.ContainsKey(entry.Key!),
|
||||
$"Unexpected file in archive: {entry.Key}"
|
||||
);
|
||||
|
||||
var expected = expectedFiles[entry.Key!];
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
await entryStream.CopyToAsync(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(expected.crc, actualCrc);
|
||||
Assert.Equal(expected.data.Length, extractedData.Length);
|
||||
|
||||
// For larger files, just spot check, for smaller verify full content
|
||||
var expectedData = expected.data;
|
||||
if (expectedData.Length <= 2 * 1024 * 1024)
|
||||
{
|
||||
Assert.Equal(expectedData, extractedData);
|
||||
}
|
||||
else
|
||||
{
|
||||
VerifyDataSpotCheck(expectedData, extractedData);
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(expectedFiles.Count, archive.Entries.Count(e => !e.IsDirectory));
|
||||
}
|
||||
}
|
||||
254
tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs
Normal file
254
tests/SharpCompress.Test/Zip/ZipReaderAsyncTests.cs
Normal file
@@ -0,0 +1,254 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Readers;
|
||||
using SharpCompress.Readers.Zip;
|
||||
using SharpCompress.Test.Mocks;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class ZipReaderAsyncTests : ReaderTests
|
||||
{
|
||||
public ZipReaderAsyncTests() => UseExtensionInsteadOfNameToVerify = true;
|
||||
|
||||
[Fact]
|
||||
public async Task Issue_269_Double_Skip_Async()
|
||||
{
|
||||
var path = Path.Combine(TEST_ARCHIVES_PATH, "PrePostHeaders.zip");
|
||||
using Stream stream = new ForwardOnlyStream(File.OpenRead(path));
|
||||
using var reader = ReaderFactory.Open(stream);
|
||||
var count = 0;
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
count++;
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
if (count % 2 != 0)
|
||||
{
|
||||
await reader.WriteEntryToAsync(Stream.Null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Zip64_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.zip64.zip", CompressionType.Deflate);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_ZipX_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.zipx", CompressionType.LZMA);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.bzip2.dd.zip", CompressionType.BZip2);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_Read_Async() =>
|
||||
await ReadAsync("Zip.bzip2.zip", CompressionType.BZip2);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Streamed2_Read_Async() =>
|
||||
await ReadAsync("Zip.deflate.dd-.zip", CompressionType.Deflate);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.deflate.dd.zip", CompressionType.Deflate);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Streamed_Skip_Async()
|
||||
{
|
||||
using Stream stream = new ForwardOnlyStream(
|
||||
File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.dd.zip"))
|
||||
);
|
||||
using var reader = ReaderFactory.Open(stream);
|
||||
var x = 0;
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
x++;
|
||||
if (x % 2 == 0)
|
||||
{
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Read_Async() =>
|
||||
await ReadAsync("Zip.deflate.zip", CompressionType.Deflate);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate64_Read_Async() =>
|
||||
await ReadAsync("Zip.deflate64.zip", CompressionType.Deflate64);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.lzma.dd.zip", CompressionType.LZMA);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_Read_Async() =>
|
||||
await ReadAsync("Zip.lzma.zip", CompressionType.LZMA);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_PPMd_Streamed_Read_Async() =>
|
||||
await ReadAsync("Zip.ppmd.dd.zip", CompressionType.PPMd);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_PPMd_Read_Async() =>
|
||||
await ReadAsync("Zip.ppmd.zip", CompressionType.PPMd);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_None_Read_Async() =>
|
||||
await ReadAsync("Zip.none.zip", CompressionType.None);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_NoEmptyDirs_Read_Async() =>
|
||||
await ReadAsync("Zip.deflate.noEmptyDirs.zip", CompressionType.Deflate);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_PkwareEncryption_Read_Async()
|
||||
{
|
||||
using (
|
||||
Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.bzip2.pkware.zip"))
|
||||
)
|
||||
using (var reader = ZipReader.Open(stream, new ReaderOptions { Password = "test" }))
|
||||
{
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
Assert.Equal(CompressionType.BZip2, reader.Entry.CompressionType);
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
VerifyFiles();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reader_Disposal_Test_Async()
|
||||
{
|
||||
using var stream = new TestStream(
|
||||
File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.dd.zip"))
|
||||
);
|
||||
using (var reader = ReaderFactory.Open(stream))
|
||||
{
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Assert.True(stream.IsDisposed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Reader_Disposal_Test2_Async()
|
||||
{
|
||||
using var stream = new TestStream(
|
||||
File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.dd.zip"))
|
||||
);
|
||||
var reader = ReaderFactory.Open(stream);
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
Assert.False(stream.IsDisposed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_WinzipAES_Read_Async() =>
|
||||
await Assert.ThrowsAsync<NotSupportedException>(async () =>
|
||||
{
|
||||
using (
|
||||
Stream stream = File.OpenRead(
|
||||
Path.Combine(TEST_ARCHIVES_PATH, "Zip.lzma.WinzipAES.zip")
|
||||
)
|
||||
)
|
||||
using (var reader = ZipReader.Open(stream, new ReaderOptions { Password = "test" }))
|
||||
{
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
Assert.Equal(CompressionType.Unknown, reader.Entry.CompressionType);
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
VerifyFiles();
|
||||
});
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_WinzipAES_Read_Async()
|
||||
{
|
||||
using (
|
||||
Stream stream = File.OpenRead(
|
||||
Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.WinzipAES.zip")
|
||||
)
|
||||
)
|
||||
using (var reader = ZipReader.Open(stream, new ReaderOptions { Password = "test" }))
|
||||
{
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
Assert.Equal(CompressionType.Unknown, reader.Entry.CompressionType);
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
VerifyFiles();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_ZipCrypto_Read_Async()
|
||||
{
|
||||
var count = 0;
|
||||
using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "zipcrypto.zip")))
|
||||
using (var reader = ZipReader.Open(stream, new ReaderOptions { Password = "test" }))
|
||||
{
|
||||
while (await reader.MoveToNextEntryAsync())
|
||||
{
|
||||
if (!reader.Entry.IsDirectory)
|
||||
{
|
||||
Assert.Equal(CompressionType.None, reader.Entry.CompressionType);
|
||||
await reader.WriteEntryToDirectoryAsync(
|
||||
SCRATCH_FILES_PATH,
|
||||
new ExtractionOptions { ExtractFullPath = true, Overwrite = true }
|
||||
);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
Assert.Equal(8, count);
|
||||
}
|
||||
}
|
||||
67
tests/SharpCompress.Test/Zip/ZipWriterAsyncTests.cs
Normal file
67
tests/SharpCompress.Test/Zip/ZipWriterAsyncTests.cs
Normal file
@@ -0,0 +1,67 @@
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using SharpCompress.Common;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class ZipWriterAsyncTests : WriterTests
|
||||
{
|
||||
public ZipWriterAsyncTests()
|
||||
: base(ArchiveType.Zip) { }
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Deflate_Write_Async() =>
|
||||
await WriteAsync(
|
||||
CompressionType.Deflate,
|
||||
"Zip.deflate.noEmptyDirs.zip",
|
||||
"Zip.deflate.noEmptyDirs.zip",
|
||||
Encoding.UTF8
|
||||
);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_BZip2_Write_Async() =>
|
||||
await WriteAsync(
|
||||
CompressionType.BZip2,
|
||||
"Zip.bzip2.noEmptyDirs.zip",
|
||||
"Zip.bzip2.noEmptyDirs.zip",
|
||||
Encoding.UTF8
|
||||
);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_None_Write_Async() =>
|
||||
await WriteAsync(
|
||||
CompressionType.None,
|
||||
"Zip.none.noEmptyDirs.zip",
|
||||
"Zip.none.noEmptyDirs.zip",
|
||||
Encoding.UTF8
|
||||
);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_LZMA_Write_Async() =>
|
||||
await WriteAsync(
|
||||
CompressionType.LZMA,
|
||||
"Zip.lzma.noEmptyDirs.zip",
|
||||
"Zip.lzma.noEmptyDirs.zip",
|
||||
Encoding.UTF8
|
||||
);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_PPMd_Write_Async() =>
|
||||
await WriteAsync(
|
||||
CompressionType.PPMd,
|
||||
"Zip.ppmd.noEmptyDirs.zip",
|
||||
"Zip.ppmd.noEmptyDirs.zip",
|
||||
Encoding.UTF8
|
||||
);
|
||||
|
||||
[Fact]
|
||||
public async Task Zip_Rar_Write_Async() =>
|
||||
await Assert.ThrowsAsync<InvalidFormatException>(async () =>
|
||||
await WriteAsync(
|
||||
CompressionType.Rar,
|
||||
"Zip.ppmd.noEmptyDirs.zip",
|
||||
"Zip.ppmd.noEmptyDirs.zip"
|
||||
)
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user