From fe4cc8e6cb8d78dfc652fb23501a98d365f365cb Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sat, 13 Feb 2021 16:44:53 +0000 Subject: [PATCH] Zip LZMA write will roundtrip --- src/SharpCompress/Common/Zip/ZipFilePart.cs | 7 +- .../Compressors/Deflate/ZlibBaseStream.cs | 26 +------- .../Compressors/LZMA/LZipStream.cs | 49 ++++---------- .../Compressors/LZMA/LzmaDecoder.cs | 65 +++++++++--------- .../Compressors/LZMA/LzmaStream.cs | 2 +- .../Compressors/LZMA/RangeCoder/RangeCoder.cs | 9 ++- .../LZMA/RangeCoder/RangeCoderBit.cs | 11 +++- .../LZMA/RangeCoder/RangeCoderBitTree.cs | 14 ++-- src/SharpCompress/IO/AsyncStream.cs | 2 - src/SharpCompress/Writers/Zip/ZipWriter.cs | 46 ++++++------- tests/SharpCompress.Test/ArchiveTests.cs | 2 +- .../GZip/GZipArchiveTests.cs | 4 +- .../GZip/GZipReaderTests.cs | 2 +- .../Streams/RewindableStreamTest.cs | 66 +++++++++---------- .../SharpCompress.Test/Tar/TarWriterTests.cs | 4 +- tests/SharpCompress.Test/Xz/XZHeaderTests.cs | 4 +- tests/SharpCompress.Test/Zip/Zip64Tests.cs | 6 +- .../SharpCompress.Test/Zip/ZipArchiveTests.cs | 6 +- .../SharpCompress.Test/Zip/ZipReaderTests.cs | 6 +- 19 files changed, 141 insertions(+), 190 deletions(-) diff --git a/src/SharpCompress/Common/Zip/ZipFilePart.cs b/src/SharpCompress/Common/Zip/ZipFilePart.cs index c09980db..3f5bb4ec 100644 --- a/src/SharpCompress/Common/Zip/ZipFilePart.cs +++ b/src/SharpCompress/Common/Zip/ZipFilePart.cs @@ -83,10 +83,9 @@ namespace SharpCompress.Common.Zip { throw new NotSupportedException("LZMA with pkware encryption."); } - var reader = new BinaryReader(stream); - reader.ReadUInt16(); //LZMA version - var props = new byte[reader.ReadUInt16()]; - reader.Read(props, 0, props.Length); + await stream.ReadUInt16(cancellationToken); //LZMA version + var props = new byte[await stream.ReadUInt16(cancellationToken)]; + await stream.ReadAsync(props, 0, props.Length, cancellationToken); return await LzmaStream.CreateAsync(props, stream, Header.CompressedSize > 0 ? Header.CompressedSize - 4 - props.Length : -1, FlagUtility.HasFlag(Header.Flags, HeaderFlags.Bit1) diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 13abc64a..c1c73a10 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -35,6 +35,7 @@ using SharpCompress.Common.Tar.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; +using SharpCompress.IO; namespace SharpCompress.Compressors.Deflate { @@ -45,7 +46,7 @@ namespace SharpCompress.Compressors.Deflate GZIP = 1952 } - internal class ZlibBaseStream : Stream + internal class ZlibBaseStream : AsyncStream { protected internal ZlibCodec _z; // deferred init... new ZlibCodec(); @@ -173,11 +174,6 @@ namespace SharpCompress.Compressors.Deflate while (!done); } - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - private async Task FinishAsync() { if (_z is null) @@ -321,14 +317,6 @@ namespace SharpCompress.Compressors.Deflate _z = null; } - protected override void Dispose(bool disposing) - { - if (disposing) - { - throw new NotImplementedException(); - } - } - public override async ValueTask DisposeAsync() { if (_isDisposed) @@ -355,11 +343,6 @@ namespace SharpCompress.Compressors.Deflate } } - public override void Flush() - { - throw new NotSupportedException(); - } - public override Task FlushAsync(CancellationToken cancellationToken) { return _stream.FlushAsync(cancellationToken); @@ -623,11 +606,6 @@ namespace SharpCompress.Compressors.Deflate return rc; } - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - public override Boolean CanRead => _stream.CanRead; public override Boolean CanSeek => _stream.CanSeek; diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.cs index 10bca9a9..247c2e01 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.cs @@ -17,9 +17,9 @@ namespace SharpCompress.Compressors.LZMA /// /// Stream supporting the LZIP format, as documented at http://www.nongnu.org/lzip/manual/lzip_manual.html /// - public sealed class LZipStream : Stream + public sealed class LZipStream : AsyncStream { - #nullable disable +#nullable disable private Stream _stream; #nullable enable private CountingWritableSubStream? _countingWritableSubStream; @@ -52,7 +52,7 @@ namespace SharpCompress.Compressors.LZMA { //default int dSize = 104 * 1024; - WriteHeaderSize(stream); + await WriteHeaderSizeAsync(stream); lzip._countingWritableSubStream = new CountingWritableSubStream(stream); lzip._stream = new Crc32Stream(new LzmaStream(new LzmaEncoderProperties(true, dSize), false, lzip._countingWritableSubStream)); @@ -73,14 +73,14 @@ namespace SharpCompress.Compressors.LZMA byte[] intBuf = new byte[8]; BinaryPrimitives.WriteUInt32LittleEndian(intBuf, crc32Stream.Crc); - _countingWritableSubStream.Write(intBuf, 0, 4); + await _countingWritableSubStream.WriteAsync(intBuf, 0, 4); BinaryPrimitives.WriteInt64LittleEndian(intBuf, _writeCount); - _countingWritableSubStream.Write(intBuf, 0, 8); + await _countingWritableSubStream.WriteAsync(intBuf, 0, 8); //total with headers BinaryPrimitives.WriteUInt64LittleEndian(intBuf, compressedCount + 6 + 20); - _countingWritableSubStream.Write(intBuf, 0, 8); + await _countingWritableSubStream.WriteAsync(intBuf, 0, 8); } _finished = true; } @@ -99,14 +99,6 @@ namespace SharpCompress.Compressors.LZMA await _stream.DisposeAsync(); } - protected override void Dispose(bool disposing) - { - if (disposing) - { - throw new NotSupportedException(); - } - } - public CompressionMode Mode { get; private set; } public override bool CanRead => Mode == CompressionMode.Decompress; @@ -115,20 +107,16 @@ namespace SharpCompress.Compressors.LZMA public override bool CanWrite => Mode == CompressionMode.Compress; - public override void Flush() + public override Task FlushAsync(CancellationToken cancellationToken) { - _stream.Flush(); + return _stream.FlushAsync(cancellationToken); } // TODO: Both Length and Position are sometimes feasible, but would require // reading the output length when we initialize. - public override long Length => throw new NotImplementedException(); + public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException(); - - public override int ReadByte() => throw new NotSupportedException(); + public override long Position { get => throw new NotImplementedException(); set => throw new NotSupportedException(); } public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = new CancellationToken()) { @@ -137,13 +125,7 @@ namespace SharpCompress.Compressors.LZMA public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotImplementedException(); - - - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } + public override void SetLength(long value) => throw new NotSupportedException(); public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = new CancellationToken()) { @@ -157,11 +139,6 @@ namespace SharpCompress.Compressors.LZMA _writeCount += count; } - public override void WriteByte(byte value) - { - throw new NotSupportedException(); - } - #endregion /// @@ -208,7 +185,7 @@ namespace SharpCompress.Compressors.LZMA private static readonly byte[] headerBytes = new byte[6] { (byte)'L', (byte)'Z', (byte)'I', (byte)'P', 1, 113 }; - public static void WriteHeaderSize(Stream stream) + public static async ValueTask WriteHeaderSizeAsync(Stream stream) { if (stream is null) { @@ -216,7 +193,7 @@ namespace SharpCompress.Compressors.LZMA } // hard coding the dictionary size encoding - stream.Write(headerBytes, 0, 6); + await stream.WriteAsync(headerBytes, 0, 6); } /// diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index b20cd6c1..d879a236 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -39,21 +39,21 @@ namespace SharpCompress.Compressors.LZMA _highCoder.Init(); } - public uint Decode(RangeCoder.Decoder rangeDecoder, uint posState) + public async ValueTask DecodeAsync(RangeCoder.Decoder rangeDecoder, uint posState) { - if (_choice.Decode(rangeDecoder) == 0) + if (await _choice.DecodeAsync(rangeDecoder) == 0) { - return _lowCoder[posState].Decode(rangeDecoder); + return await _lowCoder[posState].DecodeAsync(rangeDecoder); } uint symbol = Base.K_NUM_LOW_LEN_SYMBOLS; - if (_choice2.Decode(rangeDecoder) == 0) + if (await _choice2.DecodeAsync(rangeDecoder) == 0) { - symbol += _midCoder[posState].Decode(rangeDecoder); + symbol += await _midCoder[posState].DecodeAsync(rangeDecoder); } else { symbol += Base.K_NUM_MID_LEN_SYMBOLS; - symbol += _highCoder.Decode(rangeDecoder); + symbol += await _highCoder.DecodeAsync(rangeDecoder); } return symbol; } @@ -78,31 +78,31 @@ namespace SharpCompress.Compressors.LZMA } } - public byte DecodeNormal(RangeCoder.Decoder rangeDecoder) + public async ValueTask DecodeNormalAsync(RangeCoder.Decoder rangeDecoder) { uint symbol = 1; do { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); + symbol = (symbol << 1) | await _decoders[symbol].DecodeAsync(rangeDecoder); } while (symbol < 0x100); return (byte)symbol; } - public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, byte matchByte) + public async ValueTask DecodeWithMatchByteAsync(RangeCoder.Decoder rangeDecoder, byte matchByte) { uint symbol = 1; do { uint matchBit = (uint)(matchByte >> 7) & 1; matchByte <<= 1; - uint bit = _decoders[((1 + matchBit) << 8) + symbol].Decode(rangeDecoder); + uint bit = await _decoders[((1 + matchBit) << 8) + symbol].DecodeAsync(rangeDecoder); symbol = (symbol << 1) | bit; if (matchBit != bit) { while (symbol < 0x100) { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); + symbol = (symbol << 1) | await _decoders[symbol].DecodeAsync(rangeDecoder); } break; } @@ -149,14 +149,14 @@ namespace SharpCompress.Compressors.LZMA return ((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits)); } - public byte DecodeNormal(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte) + public ValueTask DecodeNormalAsync(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte) { - return _coders[GetState(pos, prevByte)].DecodeNormal(rangeDecoder); + return _coders[GetState(pos, prevByte)].DecodeNormalAsync(rangeDecoder); } - public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte, byte matchByte) + public ValueTask DecodeWithMatchByteAsync(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte, byte matchByte) { - return _coders[GetState(pos, prevByte)].DecodeWithMatchByte(rangeDecoder, matchByte); + return _coders[GetState(pos, prevByte)].DecodeWithMatchByteAsync(rangeDecoder, matchByte); } } @@ -292,7 +292,7 @@ namespace SharpCompress.Compressors.LZMA RangeCoder.Decoder rangeDecoder = new RangeCoder.Decoder(); await rangeDecoder.InitAsync(inStream); - Code(_dictionarySize, _outWindow, rangeDecoder); + await CodeAsync(_dictionarySize, _outWindow, rangeDecoder); _outWindow.ReleaseStream(); rangeDecoder.ReleaseStream(); @@ -308,7 +308,7 @@ namespace SharpCompress.Compressors.LZMA _outWindow = null; } - internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) + internal async ValueTask CodeAsync(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) { _literalDecoder ??= _literalDecoder.CheckNotNull(nameof(_literalDecoder)); int dictionarySizeCheck = Math.Max(dictionarySize, 1); @@ -318,19 +318,19 @@ namespace SharpCompress.Compressors.LZMA while (outWindow.HasSpace) { uint posState = (uint)outWindow._total & _posStateMask; - if (_isMatchDecoders[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState].Decode(rangeDecoder) == 0) + if (await _isMatchDecoders[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState].DecodeAsync(rangeDecoder) == 0) { byte b; byte prevByte = outWindow.GetByte(0); if (!_state.IsCharState()) { - b = _literalDecoder.DecodeWithMatchByte(rangeDecoder, + b = await _literalDecoder.DecodeWithMatchByteAsync(rangeDecoder, (uint)outWindow._total, prevByte, outWindow.GetByte((int)_rep0)); } else { - b = _literalDecoder.DecodeNormal(rangeDecoder, (uint)outWindow._total, prevByte); + b = await _literalDecoder.DecodeNormalAsync(rangeDecoder, (uint)outWindow._total, prevByte); } outWindow.PutByte(b); _state.UpdateChar(); @@ -338,13 +338,13 @@ namespace SharpCompress.Compressors.LZMA else { uint len; - if (_isRepDecoders[_state._index].Decode(rangeDecoder) == 1) + if (await _isRepDecoders[_state._index].DecodeAsync(rangeDecoder) == 1) { - if (_isRepG0Decoders[_state._index].Decode(rangeDecoder) == 0) + if (await _isRepG0Decoders[_state._index].DecodeAsync(rangeDecoder) == 0) { if ( - _isRep0LongDecoders[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState].Decode( - rangeDecoder) == 0) + await _isRep0LongDecoders[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState].DecodeAsync( + rangeDecoder) == 0) { _state.UpdateShortRep(); outWindow.PutByte(outWindow.GetByte((int)_rep0)); @@ -354,13 +354,13 @@ namespace SharpCompress.Compressors.LZMA else { UInt32 distance; - if (_isRepG1Decoders[_state._index].Decode(rangeDecoder) == 0) + if (await _isRepG1Decoders[_state._index].DecodeAsync(rangeDecoder) == 0) { distance = _rep1; } else { - if (_isRepG2Decoders[_state._index].Decode(rangeDecoder) == 0) + if (await _isRepG2Decoders[_state._index].DecodeAsync(rangeDecoder) == 0) { distance = _rep2; } @@ -374,7 +374,7 @@ namespace SharpCompress.Compressors.LZMA _rep1 = _rep0; _rep0 = distance; } - len = _repLenDecoder.Decode(rangeDecoder, posState) + Base.K_MATCH_MIN_LEN; + len = await _repLenDecoder.DecodeAsync(rangeDecoder, posState) + Base.K_MATCH_MIN_LEN; _state.UpdateRep(); } else @@ -382,23 +382,22 @@ namespace SharpCompress.Compressors.LZMA _rep3 = _rep2; _rep2 = _rep1; _rep1 = _rep0; - len = Base.K_MATCH_MIN_LEN + _lenDecoder.Decode(rangeDecoder, posState); + len = Base.K_MATCH_MIN_LEN + await _lenDecoder.DecodeAsync(rangeDecoder, posState); _state.UpdateMatch(); - uint posSlot = _posSlotDecoder[Base.GetLenToPosState(len)].Decode(rangeDecoder); + uint posSlot = await _posSlotDecoder[Base.GetLenToPosState(len)].DecodeAsync(rangeDecoder); if (posSlot >= Base.K_START_POS_MODEL_INDEX) { int numDirectBits = (int)((posSlot >> 1) - 1); _rep0 = ((2 | (posSlot & 1)) << numDirectBits); if (posSlot < Base.K_END_POS_MODEL_INDEX) { - _rep0 += BitTreeDecoder.ReverseDecode(_posDecoders, + _rep0 += await BitTreeDecoder.ReverseDecode(_posDecoders, _rep0 - posSlot - 1, rangeDecoder, numDirectBits); } else { - _rep0 += (rangeDecoder.DecodeDirectBits( - numDirectBits - Base.K_NUM_ALIGN_BITS) << Base.K_NUM_ALIGN_BITS); - _rep0 += _posAlignDecoder.ReverseDecode(rangeDecoder); + _rep0 += (await rangeDecoder.DecodeDirectBitsAsync(numDirectBits - Base.K_NUM_ALIGN_BITS) << Base.K_NUM_ALIGN_BITS); + _rep0 += await _posAlignDecoder.ReverseDecode(rangeDecoder); } } else diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 2af4959a..f6c03662 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -175,7 +175,7 @@ namespace SharpCompress.Compressors.LZMA { _inputPosition += _outWindow.CopyStream(_inputStream, toProcess); } - else if (_decoder.Code(_dictionarySize, _outWindow, _rangeDecoder) + else if (await _decoder.CodeAsync(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) { _availableBytes = _outWindow.AvailableBytes; diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs index ed2b0f5f..3cc5b564 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs @@ -190,18 +190,20 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder return _code / (_range /= total); } - public async ValueTask Decode(uint start, uint size) + public async ValueTask DecodeAsync(uint start, uint size) { _code -= start * _range; _range *= size; await NormalizeAsync(); } - public uint DecodeDirectBits(int numTotalBits) + public async ValueTask DecodeDirectBitsAsync(int numTotalBits) { uint range = _range; uint code = _code; uint result = 0; + using var byteBuffer = MemoryPool.Shared.Rent(1); + var bite = byteBuffer.Memory.Slice(0, 1); for (int i = numTotalBits; i > 0; i--) { range >>= 1; @@ -219,7 +221,8 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder if (range < K_TOP_VALUE) { - code = (code << 8) | (byte)_stream.ReadByte(); + await _stream.ReadAsync(bite); + code = (code << 8) | bite.Span[0]; range <<= 8; _total++; } diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBit.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBit.cs index 12d78691..3cd87da3 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBit.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBit.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Threading.Tasks; namespace SharpCompress.Compressors.LZMA.RangeCoder @@ -111,8 +112,10 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder _prob = K_BIT_MODEL_TOTAL >> 1; } - public uint Decode(Decoder rangeDecoder) + public async ValueTask DecodeAsync(Decoder rangeDecoder) { + using var byteBuffer = MemoryPool.Shared.Rent(1); + var bite = byteBuffer.Memory.Slice(0, 1); uint newBound = (rangeDecoder._range >> K_NUM_BIT_MODEL_TOTAL_BITS) * _prob; if (rangeDecoder._code < newBound) { @@ -120,7 +123,8 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder _prob += (K_BIT_MODEL_TOTAL - _prob) >> K_NUM_MOVE_BITS; if (rangeDecoder._range < Decoder.K_TOP_VALUE) { - rangeDecoder._code = (rangeDecoder._code << 8) | (byte)rangeDecoder._stream.ReadByte(); + await rangeDecoder._stream.ReadAsync(bite); + rangeDecoder._code = (rangeDecoder._code << 8) | bite.Span[0]; rangeDecoder._range <<= 8; rangeDecoder._total++; } @@ -131,7 +135,8 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder _prob -= (_prob) >> K_NUM_MOVE_BITS; if (rangeDecoder._range < Decoder.K_TOP_VALUE) { - rangeDecoder._code = (rangeDecoder._code << 8) | (byte)rangeDecoder._stream.ReadByte(); + await rangeDecoder._stream.ReadAsync(bite); + rangeDecoder._code = (rangeDecoder._code << 8) | bite.Span[0]; rangeDecoder._range <<= 8; rangeDecoder._total++; } diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBitTree.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBitTree.cs index 73f97a3f..022c67ea 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBitTree.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoderBitTree.cs @@ -122,23 +122,23 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder } } - public uint Decode(Decoder rangeDecoder) + public async ValueTask DecodeAsync(Decoder rangeDecoder) { uint m = 1; for (int bitIndex = _numBitLevels; bitIndex > 0; bitIndex--) { - m = (m << 1) + _models[m].Decode(rangeDecoder); + m = (m << 1) + await _models[m].DecodeAsync(rangeDecoder); } return m - ((uint)1 << _numBitLevels); } - public uint ReverseDecode(Decoder rangeDecoder) + public async ValueTask ReverseDecode(Decoder rangeDecoder) { uint m = 1; uint symbol = 0; for (int bitIndex = 0; bitIndex < _numBitLevels; bitIndex++) { - uint bit = _models[m].Decode(rangeDecoder); + uint bit = await _models[m].DecodeAsync(rangeDecoder); m <<= 1; m += bit; symbol |= (bit << bitIndex); @@ -146,14 +146,14 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder return symbol; } - public static uint ReverseDecode(BitDecoder[] models, UInt32 startIndex, - Decoder rangeDecoder, int numBitLevels) + public static async ValueTask ReverseDecode(BitDecoder[] models, UInt32 startIndex, + Decoder rangeDecoder, int numBitLevels) { uint m = 1; uint symbol = 0; for (int bitIndex = 0; bitIndex < numBitLevels; bitIndex++) { - uint bit = models[startIndex + m].Decode(rangeDecoder); + uint bit = await models[startIndex + m].DecodeAsync(rangeDecoder); m <<= 1; m += bit; symbol |= (bit << bitIndex); diff --git a/src/SharpCompress/IO/AsyncStream.cs b/src/SharpCompress/IO/AsyncStream.cs index 3d0a2a96..6a73d04e 100644 --- a/src/SharpCompress/IO/AsyncStream.cs +++ b/src/SharpCompress/IO/AsyncStream.cs @@ -54,8 +54,6 @@ namespace SharpCompress.IO public abstract override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken); - public abstract override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default); - #if !NET461 && !NETSTANDARD2_0 public sealed override int Read(Span buffer) diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index 8f214975..67a1de92 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -333,7 +333,7 @@ namespace SharpCompress.Writers.Zip #region Nested type: ZipWritingStream - private class ZipWritingStream : Stream + private class ZipWritingStream : AsyncStream { private readonly CRC32 crc = new CRC32(); private readonly ZipCentralDirectoryEntry entry; @@ -397,15 +397,21 @@ namespace SharpCompress.Writers.Zip return await BZip2Stream.CreateAsync(counting, CompressionMode.Compress, false, cancellationToken); } */ case ZipCompressionMethod.LZMA: - { - counting.WriteByte(9); - counting.WriteByte(20); - counting.WriteByte(5); - counting.WriteByte(0); + { + using var byteBuffer = MemoryPool.Shared.Rent(1); + var bite = byteBuffer.Memory.Slice(0, 1); + bite.Span[0] = 9; + await counting.WriteAsync(bite, cancellationToken); + bite.Span[0] = 20; + await counting.WriteAsync(bite, cancellationToken); + bite.Span[0] = 5; + await counting.WriteAsync(bite, cancellationToken); + bite.Span[0] = 0; + await counting.WriteAsync(bite, cancellationToken); LzmaStream lzmaStream = new LzmaStream(new LzmaEncoderProperties(!originalStream.CanSeek), false, counting); - counting.Write(lzmaStream.Properties, 0, lzmaStream.Properties.Length); + await counting.WriteAsync(lzmaStream.Properties, 0, lzmaStream.Properties.Length, cancellationToken); return lzmaStream; } /* case ZipCompressionMethod.PPMd: @@ -429,7 +435,6 @@ namespace SharpCompress.Writers.Zip isDisposed = true; - await base.DisposeAsync(); await writeStream.DisposeAsync(); if (limitsExceeded) @@ -452,14 +457,17 @@ namespace SharpCompress.Writers.Zip if (originalStream.CanSeek) { originalStream.Position = (long)(entry.HeaderOffset + 6); - originalStream.WriteByte(0); + using var byteBuffer = MemoryPool.Shared.Rent(1); + var bite = byteBuffer.Memory.Slice(0, 1); + bite.Span[0] = 0; + await originalStream.WriteAsync(bite); if (counting.Count == 0 && entry.Decompressed == 0) { // set compression to STORED for zero byte files (no compression data) originalStream.Position = (long)(entry.HeaderOffset + 8); - originalStream.WriteByte(0); - originalStream.WriteByte(0); + await originalStream.WriteAsync(bite); + await originalStream.WriteAsync(bite); } originalStream.Position = (long)(entry.HeaderOffset + 14); @@ -517,21 +525,10 @@ namespace SharpCompress.Writers.Zip writer.entries.Add(entry); } - public override void Flush() - { - writeStream.Flush(); - } - public override Task FlushAsync(CancellationToken cancellationToken) { return writeStream.FlushAsync(cancellationToken); } - - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } - public override long Seek(long offset, SeekOrigin origin) { throw new NotSupportedException(); @@ -542,11 +539,6 @@ namespace SharpCompress.Writers.Zip throw new NotSupportedException(); } - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { // We check the limits first, because we can keep the archive consistent diff --git a/tests/SharpCompress.Test/ArchiveTests.cs b/tests/SharpCompress.Test/ArchiveTests.cs index 7320c3d5..2e20b414 100644 --- a/tests/SharpCompress.Test/ArchiveTests.cs +++ b/tests/SharpCompress.Test/ArchiveTests.cs @@ -69,7 +69,7 @@ namespace SharpCompress.Test { foreach (var path in testArchives) { - using (var stream = new NonDisposingStream(File.OpenRead(path), true)) + await using (var stream = new NonDisposingStream(File.OpenRead(path), true)) await using (var archive = await ArchiveFactory.OpenAsync(stream, readerOptions)) { try diff --git a/tests/SharpCompress.Test/GZip/GZipArchiveTests.cs b/tests/SharpCompress.Test/GZip/GZipArchiveTests.cs index feb35030..fb095f7a 100644 --- a/tests/SharpCompress.Test/GZip/GZipArchiveTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipArchiveTests.cs @@ -18,7 +18,7 @@ namespace SharpCompress.Test.GZip [Fact] public async ValueTask GZip_Archive_Generic() { - using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) + await using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) await using (var archive = await ArchiveFactory.OpenAsync(stream)) { var entry = await archive.Entries.FirstAsync(); @@ -38,7 +38,7 @@ namespace SharpCompress.Test.GZip [Fact] public async ValueTask GZip_Archive() { - using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) + await using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) await using (var archive = GZipArchive.Open(stream)) { var entry = await archive.Entries.FirstAsync(); diff --git a/tests/SharpCompress.Test/GZip/GZipReaderTests.cs b/tests/SharpCompress.Test/GZip/GZipReaderTests.cs index 84ca11b0..3913c0df 100644 --- a/tests/SharpCompress.Test/GZip/GZipReaderTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipReaderTests.cs @@ -24,7 +24,7 @@ namespace SharpCompress.Test.GZip public async ValueTask GZip_Reader_Generic2() { //read only as GZip itme - using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) + await using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"))) await using (var reader = SharpCompress.Readers.GZip.GZipReader.Open(new RewindableStream(stream))) { while (await reader.MoveToNextEntryAsync()) // Crash here diff --git a/tests/SharpCompress.Test/Streams/RewindableStreamTest.cs b/tests/SharpCompress.Test/Streams/RewindableStreamTest.cs index 14beab4f..8f3467da 100644 --- a/tests/SharpCompress.Test/Streams/RewindableStreamTest.cs +++ b/tests/SharpCompress.Test/Streams/RewindableStreamTest.cs @@ -1,4 +1,6 @@ using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.IO; using Xunit; @@ -7,7 +9,7 @@ namespace SharpCompress.Test.Streams public class RewindableStreamTest { [Fact] - public void TestRewind() + public async Task TestRewind() { MemoryStream ms = new MemoryStream(); BinaryWriter bw = new BinaryWriter(ms); @@ -22,30 +24,29 @@ namespace SharpCompress.Test.Streams ms.Position = 0; RewindableStream stream = new RewindableStream(ms); stream.StartRecording(); - BinaryReader br = new BinaryReader(stream); - Assert.Equal(1, br.ReadInt32()); - Assert.Equal(2, br.ReadInt32()); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); + Assert.Equal(1, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(2, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); stream.Rewind(true); stream.StartRecording(); - Assert.Equal(1, br.ReadInt32()); - Assert.Equal(2, br.ReadInt32()); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); - Assert.Equal(5, br.ReadInt32()); - Assert.Equal(6, br.ReadInt32()); - Assert.Equal(7, br.ReadInt32()); + Assert.Equal(1, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(2, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(5, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(6, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(7, await stream.ReadInt32(CancellationToken.None)); stream.Rewind(true); stream.StartRecording(); - Assert.Equal(1, br.ReadInt32()); - Assert.Equal(2, br.ReadInt32()); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); + Assert.Equal(1, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(2, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); } [Fact] - public void TestIncompleteRewind() + public async Task TestIncompleteRewind() { MemoryStream ms = new MemoryStream(); BinaryWriter bw = new BinaryWriter(ms); @@ -60,24 +61,23 @@ namespace SharpCompress.Test.Streams ms.Position = 0; RewindableStream stream = new RewindableStream(ms); stream.StartRecording(); - BinaryReader br = new BinaryReader(stream); - Assert.Equal(1, br.ReadInt32()); - Assert.Equal(2, br.ReadInt32()); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); + Assert.Equal(1, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(2, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); stream.Rewind(true); - Assert.Equal(1, br.ReadInt32()); - Assert.Equal(2, br.ReadInt32()); + Assert.Equal(1, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(2, await stream.ReadInt32(CancellationToken.None)); stream.StartRecording(); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); - Assert.Equal(5, br.ReadInt32()); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(5, await stream.ReadInt32(CancellationToken.None)); stream.Rewind(true); - Assert.Equal(3, br.ReadInt32()); - Assert.Equal(4, br.ReadInt32()); - Assert.Equal(5, br.ReadInt32()); - Assert.Equal(6, br.ReadInt32()); - Assert.Equal(7, br.ReadInt32()); + Assert.Equal(3, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(4, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(5, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(6, await stream.ReadInt32(CancellationToken.None)); + Assert.Equal(7, await stream.ReadInt32(CancellationToken.None)); } } } diff --git a/tests/SharpCompress.Test/Tar/TarWriterTests.cs b/tests/SharpCompress.Test/Tar/TarWriterTests.cs index 97c07cc1..892cb4e1 100644 --- a/tests/SharpCompress.Test/Tar/TarWriterTests.cs +++ b/tests/SharpCompress.Test/Tar/TarWriterTests.cs @@ -44,8 +44,8 @@ namespace SharpCompress.Test.Tar [InlineData(false)] public async Task Tar_Finalize_Archive(bool finalizeArchive) { - using (MemoryStream stream = new MemoryStream()) - using (Stream content = File.OpenRead(Path.Combine(ORIGINAL_FILES_PATH, "jpg", "test.jpg"))) + await using (MemoryStream stream = new MemoryStream()) + await using (Stream content = File.OpenRead(Path.Combine(ORIGINAL_FILES_PATH, "jpg", "test.jpg"))) { await using (TarWriter writer = await TarWriter.CreateAsync(stream, new TarWriterOptions(CompressionType.None, finalizeArchive))) { diff --git a/tests/SharpCompress.Test/Xz/XZHeaderTests.cs b/tests/SharpCompress.Test/Xz/XZHeaderTests.cs index 91266fc8..d9b53098 100644 --- a/tests/SharpCompress.Test/Xz/XZHeaderTests.cs +++ b/tests/SharpCompress.Test/Xz/XZHeaderTests.cs @@ -25,7 +25,7 @@ namespace SharpCompress.Test.Xz { var bytes = Compressed.Clone() as byte[]; bytes[8]++; - using (Stream badCrcStream = new MemoryStream(bytes)) + await using (Stream badCrcStream = new MemoryStream(bytes)) { var header = new XZHeader(badCrcStream); var ex = await Assert.ThrowsAsync(async () => { await header.Process(); }); @@ -41,7 +41,7 @@ namespace SharpCompress.Test.Xz byte[] crc = Crc32.Compute(streamFlags).ToLittleEndianBytes(); streamFlags.CopyTo(bytes, 6); crc.CopyTo(bytes, 8); - using (Stream badFlagStream = new MemoryStream(bytes)) + await using (Stream badFlagStream = new MemoryStream(bytes)) { var header = new XZHeader(badFlagStream); var ex = await Assert.ThrowsAsync(async () => { await header.Process(); }); diff --git a/tests/SharpCompress.Test/Zip/Zip64Tests.cs b/tests/SharpCompress.Test/Zip/Zip64Tests.cs index 96468713..5e745f33 100644 --- a/tests/SharpCompress.Test/Zip/Zip64Tests.cs +++ b/tests/SharpCompress.Test/Zip/Zip64Tests.cs @@ -147,8 +147,8 @@ namespace SharpCompress.Test.Zip // Use no compression to ensure we hit the limits (actually inflates a bit, but seems better than using method==Store) var eo = new ZipWriterEntryOptions() { DeflateCompressionLevel = Compressors.Deflate.CompressionLevel.None }; - using (var zip = File.OpenWrite(filename)) - using (var st = forward_only ? (Stream)new ForwardOnlyStream(zip) : zip) + await using (var zip = File.OpenWrite(filename)) + await using (var st = forward_only ? (Stream)new ForwardOnlyStream(zip) : zip) await using (var zipWriter = (ZipWriter)await WriterFactory.OpenAsync(st, ArchiveType.Zip, opts)) { @@ -173,7 +173,7 @@ namespace SharpCompress.Test.Zip long count = 0; long size = 0; Common.Zip.ZipEntry prev = null; - using (var fs = File.OpenRead(filename)) + await using (var fs = File.OpenRead(filename)) await using (var rd = ZipReader.Open(fs, new ReaderOptions() { LookForHeader = false })) { while (await rd.MoveToNextEntryAsync()) diff --git a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs index 4901e645..02ee2a9c 100644 --- a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs @@ -529,7 +529,7 @@ namespace SharpCompress.Test.Zip var buffer = new byte[4096]; await using (var memoryStream = new MemoryStream()) - using (var firstStream = await firstEntry.OpenEntryStreamAsync()) + await using (var firstStream = await firstEntry.OpenEntryStreamAsync()) { await firstStream.CopyToAsync(memoryStream); Assert.Equal(199, memoryStream.Length); @@ -550,8 +550,8 @@ namespace SharpCompress.Test.Zip var firstEntry = await za.Entries.FirstAsync(x => x.Key == "first.txt"); var buffer = new byte[4096]; - using (var memoryStream = new MemoryStream()) - using (var firstStream = await firstEntry.OpenEntryStreamAsync()) + await using (var memoryStream = new MemoryStream()) + await using (var firstStream = await firstEntry.OpenEntryStreamAsync()) { await firstStream.CopyToAsync(memoryStream); Assert.Equal(199, memoryStream.Length); diff --git a/tests/SharpCompress.Test/Zip/ZipReaderTests.cs b/tests/SharpCompress.Test/Zip/ZipReaderTests.cs index 59f3a67c..0fa8cf29 100644 --- a/tests/SharpCompress.Test/Zip/ZipReaderTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipReaderTests.cs @@ -194,7 +194,7 @@ namespace SharpCompress.Test.Zip [Fact] public async ValueTask Zip_Reader_Disposal_Test2() { - using (TestStream stream = new TestStream(File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.dd.zip")))) + await using (TestStream stream = new TestStream(File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.deflate.dd.zip")))) { var reader = await ReaderFactory.OpenAsync(stream); while (await reader.MoveToNextEntryAsync()) @@ -218,7 +218,7 @@ namespace SharpCompress.Test.Zip { await Assert.ThrowsAsync(async () => { - using ( + await using ( Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.lzma.WinzipAES.zip"))) @@ -316,7 +316,7 @@ namespace SharpCompress.Test.Zip { var keys = new string[] { "Empty1", "Empty2", "Dir1/", "Dir2/", "Fake1", "Fake2", "Internal.zip" }; - using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.none.issue86.zip"))) + await using (Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Zip.none.issue86.zip"))) await using (var reader = ZipReader.Open(stream)) { foreach (var key in keys)