From ef3d4da28637af49e97f52b4470ff91bacac2ed3 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 8 Feb 2021 11:18:57 +0000 Subject: [PATCH] Fix test and some zip writing --- src/SharpCompress/Archives/Zip/ZipArchive.cs | 12 ++++- .../Common/Zip/StreamingZipHeaderFactory.cs | 32 ++++-------- .../Compressors/Deflate/DeflateStream.cs | 20 ++++---- .../Compressors/Deflate/ZlibBaseStream.cs | 26 +++++++--- .../IO/CountingWritableSubStream.cs | 14 ++++++ src/SharpCompress/IO/NonDisposingStream.cs | 14 +++++- src/SharpCompress/IO/RewindableStream.cs | 37 ++++++++++++++ src/SharpCompress/Readers/Zip/ZipReader.cs | 12 ++++- .../Writers/Zip/ZipCentralDirectoryEntry.cs | 50 ++++++++++--------- src/SharpCompress/Writers/Zip/ZipWriter.cs | 4 +- 10 files changed, 149 insertions(+), 72 deletions(-) diff --git a/src/SharpCompress/Archives/Zip/ZipArchive.cs b/src/SharpCompress/Archives/Zip/ZipArchive.cs index 119e5e86..75b7106c 100644 --- a/src/SharpCompress/Archives/Zip/ZipArchive.cs +++ b/src/SharpCompress/Archives/Zip/ZipArchive.cs @@ -9,6 +9,7 @@ using SharpCompress.Common; using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; using SharpCompress.Compressors.Deflate; +using SharpCompress.IO; using SharpCompress.Readers; using SharpCompress.Readers.Zip; using SharpCompress.Writers; @@ -84,7 +85,16 @@ namespace SharpCompress.Archives.Zip StreamingZipHeaderFactory headerFactory = new(password, new ArchiveEncoding()); try { - ZipHeader? header = await headerFactory.ReadStreamHeader(stream, cancellationToken) + RewindableStream rewindableStream; + if (stream is RewindableStream rs) + { + rewindableStream = rs; + } + else + { + rewindableStream = new RewindableStream(stream); + } + ZipHeader? header = await headerFactory.ReadStreamHeader(rewindableStream, cancellationToken) .FirstOrDefaultAsync(x => x.ZipHeaderType != ZipHeaderType.Split, cancellationToken: cancellationToken); if (header is null) { diff --git a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs index cfb713d3..339ac23b 100644 --- a/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs @@ -1,8 +1,6 @@ using System.Collections.Generic; -using System.IO; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; using SharpCompress.Common.Zip.Headers; using SharpCompress.IO; @@ -15,20 +13,8 @@ namespace SharpCompress.Common.Zip { } - internal async IAsyncEnumerable ReadStreamHeader(Stream stream, [EnumeratorCancellation] CancellationToken cancellationToken) + internal async IAsyncEnumerable ReadStreamHeader(RewindableStream rewindableStream, [EnumeratorCancellation] CancellationToken cancellationToken) { - //TODO async stream reader? - await Task.CompletedTask; - RewindableStream rewindableStream; - - if (stream is RewindableStream rs) - { - rewindableStream = rs; - } - else - { - rewindableStream = new RewindableStream(stream); - } while (true) { ZipHeader? header; @@ -37,22 +23,22 @@ namespace SharpCompress.Common.Zip { await ((StreamingZipFilePart)_lastEntryHeader.Part).FixStreamedFileLocation(rewindableStream, cancellationToken); long? pos = rewindableStream.CanSeek ? (long?)rewindableStream.Position : null; - uint crc = await stream.ReadUInt32(cancellationToken); + uint crc = await rewindableStream.ReadUInt32(cancellationToken); if (crc == POST_DATA_DESCRIPTOR) { - crc = await stream.ReadUInt32(cancellationToken); + crc = await rewindableStream.ReadUInt32(cancellationToken); } _lastEntryHeader.Crc = crc; - _lastEntryHeader.CompressedSize = await stream.ReadUInt32(cancellationToken); - _lastEntryHeader.UncompressedSize = await stream.ReadUInt32(cancellationToken); + _lastEntryHeader.CompressedSize = await rewindableStream.ReadUInt32(cancellationToken); + _lastEntryHeader.UncompressedSize = await rewindableStream.ReadUInt32(cancellationToken); if (pos.HasValue) { _lastEntryHeader.DataStartPosition = pos - _lastEntryHeader.CompressedSize; } } _lastEntryHeader = null; - uint headerBytes = await stream.ReadUInt32(cancellationToken); - header = await ReadHeader(headerBytes, stream, cancellationToken); + uint headerBytes = await rewindableStream.ReadUInt32(cancellationToken); + header = await ReadHeader(headerBytes, rewindableStream, cancellationToken); if (header is null) { yield break; @@ -75,10 +61,10 @@ namespace SharpCompress.Common.Zip { rewindableStream.StartRecording(); } - uint nextHeaderBytes = await stream.ReadUInt32(cancellationToken); + uint nextHeaderBytes = await rewindableStream.ReadUInt32(cancellationToken); // Check if next data is PostDataDescriptor, streamed file with 0 length - header.HasData = !IsHeader(nextHeaderBytes); + header.HasData = nextHeaderBytes != POST_DATA_DESCRIPTOR; rewindableStream.Rewind(!isRecording); } else // We are not streaming and compressed size is 0, we have no data diff --git a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs index 74c152b5..a324400d 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs @@ -220,20 +220,18 @@ namespace SharpCompress.Compressors.Deflate /// protected override void Dispose(bool disposing) { - try + if (disposing) { - if (!_disposed) - { - if (disposing) - { - _baseStream?.Dispose(); - } - _disposed = true; - } + throw new NotImplementedException(); } - finally + } + + public override async ValueTask DisposeAsync() + { + if (!_disposed) { - base.Dispose(disposing); + await _baseStream.DisposeAsync(); + _disposed = true; } } diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 10c45bea..13abc64a 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -321,6 +321,14 @@ namespace SharpCompress.Compressors.Deflate _z = null; } + protected override void Dispose(bool disposing) + { + if (disposing) + { + throw new NotImplementedException(); + } + } + public override async ValueTask DisposeAsync() { if (_isDisposed) @@ -328,7 +336,6 @@ namespace SharpCompress.Compressors.Deflate return; } _isDisposed = true; - await base.DisposeAsync(); if (_stream is null) { return; @@ -340,7 +347,10 @@ namespace SharpCompress.Compressors.Deflate finally { End(); - _stream?.Dispose(); + if (_stream is not null) + { + await _stream.DisposeAsync(); + } _stream = null; } } @@ -408,13 +418,13 @@ namespace SharpCompress.Compressors.Deflate return _encoding.GetString(buffer, 0, buffer.Length); } - private async Task ReadAndValidateGzipHeaderAsync() + private async Task ReadAndValidateGzipHeaderAsync(CancellationToken cancellationToken) { var totalBytesRead = 0; // read the header on the first read using var rented = MemoryPool.Shared.Rent(10); - int n = await _stream.ReadAsync(rented.Memory.Slice(0,10)); + int n = await _stream.ReadAsync(rented.Memory.Slice(0,10), cancellationToken); var header = rented.Memory; // workitem 8501: handle edge case (decompress empty stream) @@ -439,12 +449,12 @@ namespace SharpCompress.Compressors.Deflate if ((header.Span[3] & 0x04) == 0x04) { // read and discard extra field - n = _stream.Read(header.Span.Slice(0, 2)); // 2-byte length field + n = await _stream.ReadAsync(header.Slice(0, 2), cancellationToken); // 2-byte length field totalBytesRead += n; var extraLength = (short)(header.Span[0] + header.Span[1] * 256); using var extra = MemoryPool.Shared.Rent(extraLength); - n = await _stream.ReadAsync(extra.Memory.Slice(0, extraLength)); + n = await _stream.ReadAsync(extra.Memory.Slice(0, extraLength), cancellationToken); if (n != extraLength) { throw new ZlibException("Unexpected end-of-file reading GZIP header."); @@ -461,7 +471,7 @@ namespace SharpCompress.Compressors.Deflate } if ((header.Span[3] & 0x02) == 0x02) { - await ReadAsync(_buf1, 0, 1); // CRC16, ignore + await ReadAsync(_buf1, 0, 1, cancellationToken); // CRC16, ignore } return totalBytesRead; @@ -490,7 +500,7 @@ namespace SharpCompress.Compressors.Deflate z.AvailableBytesIn = 0; if (_flavor == ZlibStreamFlavor.GZIP) { - _gzipHeaderByteCount = await ReadAndValidateGzipHeaderAsync(); + _gzipHeaderByteCount = await ReadAndValidateGzipHeaderAsync(cancellationToken); // workitem 8501: handle edge case (decompress empty stream) if (_gzipHeaderByteCount == 0) diff --git a/src/SharpCompress/IO/CountingWritableSubStream.cs b/src/SharpCompress/IO/CountingWritableSubStream.cs index b94303bf..02f175a2 100644 --- a/src/SharpCompress/IO/CountingWritableSubStream.cs +++ b/src/SharpCompress/IO/CountingWritableSubStream.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; namespace SharpCompress.IO { @@ -41,6 +43,18 @@ namespace SharpCompress.IO throw new NotSupportedException(); } + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await Stream.WriteAsync(buffer, offset, count, cancellationToken); + Count += (uint)buffer.Length; + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await Stream.WriteAsync(buffer, cancellationToken); + Count += (uint)buffer.Length; + } + public override void Write(byte[] buffer, int offset, int count) { Stream.Write(buffer, offset, count); diff --git a/src/SharpCompress/IO/NonDisposingStream.cs b/src/SharpCompress/IO/NonDisposingStream.cs index ceb041e4..9eabcbb9 100644 --- a/src/SharpCompress/IO/NonDisposingStream.cs +++ b/src/SharpCompress/IO/NonDisposingStream.cs @@ -55,11 +55,21 @@ namespace SharpCompress.IO throw new NotImplementedException(); } - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = new CancellationToken()) + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { return Stream.ReadAsync(buffer, cancellationToken); } + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) + { + throw new NotImplementedException(); + } + + public override int EndRead(IAsyncResult asyncResult) + { + throw new NotImplementedException(); + } + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { return Stream.ReadAsync(buffer, offset, count, cancellationToken); @@ -82,7 +92,7 @@ namespace SharpCompress.IO public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - throw new NotImplementedException(); + return Stream.WriteAsync(buffer, offset, count, cancellationToken); } public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = new CancellationToken()) diff --git a/src/SharpCompress/IO/RewindableStream.cs b/src/SharpCompress/IO/RewindableStream.cs index 9aee8ba7..f286fbba 100644 --- a/src/SharpCompress/IO/RewindableStream.cs +++ b/src/SharpCompress/IO/RewindableStream.cs @@ -115,6 +115,43 @@ namespace SharpCompress.IO throw new NotImplementedException(); } + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + //don't actually read if we don't really want to read anything + //currently a network stream bug on Windows for .NET Core + if (count == 0) + { + return 0; + } + int read; + if (isRewound && bufferStream.Position != bufferStream.Length) + { + read = await bufferStream.ReadAsync(buffer, offset, count, cancellationToken); + if (read < count) + { + int tempRead = await stream.ReadAsync(buffer, read, count - read, cancellationToken); + if (IsRecording) + { + await bufferStream.WriteAsync(buffer, read, tempRead, cancellationToken); + } + read += tempRead; + } + if (bufferStream.Position == bufferStream.Length && !IsRecording) + { + isRewound = false; + bufferStream.SetLength(0); + } + return read; + } + + read = await stream.ReadAsync(buffer, cancellationToken); + if (IsRecording) + { + await bufferStream.WriteAsync(buffer, cancellationToken); + } + return read; + } + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { var count = buffer.Length; diff --git a/src/SharpCompress/Readers/Zip/ZipReader.cs b/src/SharpCompress/Readers/Zip/ZipReader.cs index 8d3b0b6f..5393cd60 100644 --- a/src/SharpCompress/Readers/Zip/ZipReader.cs +++ b/src/SharpCompress/Readers/Zip/ZipReader.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; +using SharpCompress.IO; namespace SharpCompress.Readers.Zip { @@ -40,7 +41,16 @@ namespace SharpCompress.Readers.Zip protected override async IAsyncEnumerable GetEntries(Stream stream, [EnumeratorCancellation] CancellationToken cancellationToken) { - await foreach (ZipHeader h in _headerFactory.ReadStreamHeader(stream, cancellationToken).WithCancellation(cancellationToken)) + RewindableStream rewindableStream; + if (stream is RewindableStream rs) + { + rewindableStream = rs; + } + else + { + rewindableStream = new RewindableStream(stream); + } + await foreach (ZipHeader h in _headerFactory.ReadStreamHeader(rewindableStream, cancellationToken).WithCancellation(cancellationToken)) { if (h != null) { diff --git a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs index d936c352..ee34d575 100644 --- a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs +++ b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs @@ -2,6 +2,8 @@ using System.Buffers.Binary; using System.IO; using System.Text; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Zip; using SharpCompress.Common.Zip.Headers; @@ -30,7 +32,7 @@ namespace SharpCompress.Writers.Zip internal ushort Zip64HeaderOffset { get; set; } internal ulong HeaderOffset { get; } - internal uint Write(Stream outputStream) + internal async ValueTask WriteAsync(Stream outputStream, CancellationToken cancellationToken) { byte[] encodedFilename = archiveEncoding.Encode(fileName); byte[] encodedComment = archiveEncoding.Encode(Comment ?? string.Empty); @@ -73,61 +75,61 @@ namespace SharpCompress.Writers.Zip byte[] intBuf = new byte[] { 80, 75, 1, 2, version, 0, version, 0 }; //constant sig, then version made by, then version to extract - outputStream.Write(intBuf, 0, 8); + await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken); BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags); - outputStream.Write(intBuf, 0, 2); + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)usedCompression); - outputStream.Write(intBuf, 0, 2); // zipping method + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // zipping method BinaryPrimitives.WriteUInt32LittleEndian(intBuf, ModificationTime.DateTimeToDosTime()); - outputStream.Write(intBuf, 0, 4); + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // zipping date and time BinaryPrimitives.WriteUInt32LittleEndian(intBuf, Crc); - outputStream.Write(intBuf, 0, 4); // file CRC + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // file CRC BinaryPrimitives.WriteUInt32LittleEndian(intBuf, compressedvalue); - outputStream.Write(intBuf, 0, 4); // compressed file size + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // compressed file size BinaryPrimitives.WriteUInt32LittleEndian(intBuf, decompressedvalue); - outputStream.Write(intBuf, 0, 4); // uncompressed file size + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // uncompressed file size BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)encodedFilename.Length); - outputStream.Write(intBuf, 0, 2); // Filename in zip + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // Filename in zip BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)extralength); - outputStream.Write(intBuf, 0, 2); // extra length + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // extra length BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)encodedComment.Length); - outputStream.Write(intBuf, 0, 2); + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0); - outputStream.Write(intBuf, 0, 2); // disk=0 + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // disk=0 BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags); - outputStream.Write(intBuf, 0, 2); // file type: binary + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // file type: binary BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags); - outputStream.Write(intBuf, 0, 2); // Internal file attributes + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // Internal file attributes BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0x8100); - outputStream.Write(intBuf, 0, 2); + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // External file attributes (normal/readable) BinaryPrimitives.WriteUInt32LittleEndian(intBuf, headeroffsetvalue); - outputStream.Write(intBuf, 0, 4); // Offset of header + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // Offset of header - outputStream.Write(encodedFilename, 0, encodedFilename.Length); + await outputStream.WriteAsync(encodedFilename, 0, encodedFilename.Length, cancellationToken); if (zip64) { BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0x0001); - outputStream.Write(intBuf, 0, 2); + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)(extralength - 4)); - outputStream.Write(intBuf, 0, 2); + await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); BinaryPrimitives.WriteUInt64LittleEndian(intBuf, Decompressed); - outputStream.Write(intBuf, 0, 8); + await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken); BinaryPrimitives.WriteUInt64LittleEndian(intBuf, Compressed); - outputStream.Write(intBuf, 0, 8); + await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken); BinaryPrimitives.WriteUInt64LittleEndian(intBuf, HeaderOffset); - outputStream.Write(intBuf, 0, 8); + await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken); BinaryPrimitives.WriteUInt32LittleEndian(intBuf, 0); - outputStream.Write(intBuf, 0, 4); // VolumeNumber = 0 + await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // VolumeNumber = 0 } - outputStream.Write(encodedComment, 0, encodedComment.Length); + await outputStream.WriteAsync(encodedComment, 0, encodedComment.Length, cancellationToken); return (uint)(8 + 2 + 2 + 4 + 4 + 4 + 4 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + extralength + encodedComment.Length); diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index e865901e..eec31143 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -22,7 +22,7 @@ namespace SharpCompress.Writers.Zip { private readonly CompressionType compressionType; private readonly CompressionLevel compressionLevel; - private readonly List entries = new List(); + private readonly List entries = new(); private readonly string zipComment; private long streamPosition; private PpmdProperties? ppmdProps; @@ -61,7 +61,7 @@ namespace SharpCompress.Writers.Zip ulong size = 0; foreach (ZipCentralDirectoryEntry entry in entries) { - size += entry.Write(OutputStream); + size += await entry.WriteAsync(OutputStream, CancellationToken.None); } await WriteEndRecordAsync(size); await base.DisposeAsyncCore();