starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-25 08:24:58 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Fix test and some zip writing

Browse Source
This commit is contained in:
Adam Hathcock
2021-02-08 11:18:57 +00:00
parent 813bd5ae80
commit ef3d4da286
10 changed files with 149 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/SharpCompress/Archives/Zip/ZipArchive.cs
View File

@@ -9,6 +9,7 @@ using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.Compressors.Deflate;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.Zip;
using SharpCompress.Writers;
@@ -84,7 +85,16 @@ namespace SharpCompress.Archives.Zip
StreamingZipHeaderFactory headerFactory = new(password, new ArchiveEncoding());
try
{
ZipHeader? header = await headerFactory.ReadStreamHeader(stream, cancellationToken)
RewindableStream rewindableStream;
if (stream is RewindableStream rs)
{
rewindableStream = rs;
}
else
{
rewindableStream = new RewindableStream(stream);
}
ZipHeader? header = await headerFactory.ReadStreamHeader(rewindableStream, cancellationToken)
.FirstOrDefaultAsync(x => x.ZipHeaderType != ZipHeaderType.Split, cancellationToken: cancellationToken);
if (header is null)
{

32
src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs
View File

@@ -1,8 +1,6 @@
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
@@ -15,20 +13,8 @@ namespace SharpCompress.Common.Zip
{
}
internal async IAsyncEnumerable<ZipHeader> ReadStreamHeader(Stream stream, [EnumeratorCancellation] CancellationToken cancellationToken)
internal async IAsyncEnumerable<ZipHeader> ReadStreamHeader(RewindableStream rewindableStream, [EnumeratorCancellation] CancellationToken cancellationToken)
{
//TODO async stream reader?
await Task.CompletedTask;
RewindableStream rewindableStream;
if (stream is RewindableStream rs)
{
rewindableStream = rs;
}
else
{
rewindableStream = new RewindableStream(stream);
}
while (true)
{
ZipHeader? header;
@@ -37,22 +23,22 @@ namespace SharpCompress.Common.Zip
{
await ((StreamingZipFilePart)_lastEntryHeader.Part).FixStreamedFileLocation(rewindableStream, cancellationToken);
long? pos = rewindableStream.CanSeek ? (long?)rewindableStream.Position : null;
uint crc = await stream.ReadUInt32(cancellationToken);
uint crc = await rewindableStream.ReadUInt32(cancellationToken);
if (crc == POST_DATA_DESCRIPTOR)
{
crc = await stream.ReadUInt32(cancellationToken);
crc = await rewindableStream.ReadUInt32(cancellationToken);
}
_lastEntryHeader.Crc = crc;
_lastEntryHeader.CompressedSize = await stream.ReadUInt32(cancellationToken);
_lastEntryHeader.UncompressedSize = await stream.ReadUInt32(cancellationToken);
_lastEntryHeader.CompressedSize = await rewindableStream.ReadUInt32(cancellationToken);
_lastEntryHeader.UncompressedSize = await rewindableStream.ReadUInt32(cancellationToken);
if (pos.HasValue)
{
_lastEntryHeader.DataStartPosition = pos - _lastEntryHeader.CompressedSize;
}
}
_lastEntryHeader = null;
uint headerBytes = await stream.ReadUInt32(cancellationToken);
header = await ReadHeader(headerBytes, stream, cancellationToken);
uint headerBytes = await rewindableStream.ReadUInt32(cancellationToken);
header = await ReadHeader(headerBytes, rewindableStream, cancellationToken);
if (header is null)
{
yield break;
@@ -75,10 +61,10 @@ namespace SharpCompress.Common.Zip
{
rewindableStream.StartRecording();
}
uint nextHeaderBytes = await stream.ReadUInt32(cancellationToken);
uint nextHeaderBytes = await rewindableStream.ReadUInt32(cancellationToken);
// Check if next data is PostDataDescriptor, streamed file with 0 length
header.HasData = !IsHeader(nextHeaderBytes);
header.HasData = nextHeaderBytes != POST_DATA_DESCRIPTOR;
rewindableStream.Rewind(!isRecording);
}
else // We are not streaming and compressed size is 0, we have no data

20
src/SharpCompress/Compressors/Deflate/DeflateStream.cs
View File

@@ -220,20 +220,18 @@ namespace SharpCompress.Compressors.Deflate
/// </remarks>
protected override void Dispose(bool disposing)
{
try
if (disposing)
{
if (!_disposed)
{
if (disposing)
{
_baseStream?.Dispose();
}
_disposed = true;
}
throw new NotImplementedException();
}
finally
}
public override async ValueTask DisposeAsync()
{
if (!_disposed)
{
base.Dispose(disposing);
await _baseStream.DisposeAsync();
_disposed = true;
}
}

26
src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs
View File

@@ -321,6 +321,14 @@ namespace SharpCompress.Compressors.Deflate
_z = null;
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
throw new NotImplementedException();
}
}
public override async ValueTask DisposeAsync()
{
if (_isDisposed)
@@ -328,7 +336,6 @@ namespace SharpCompress.Compressors.Deflate
return;
}
_isDisposed = true;
await base.DisposeAsync();
if (_stream is null)
{
return;
@@ -340,7 +347,10 @@ namespace SharpCompress.Compressors.Deflate
finally
{
End();
_stream?.Dispose();
if (_stream is not null)
{
await _stream.DisposeAsync();
}
_stream = null;
}
}
@@ -408,13 +418,13 @@ namespace SharpCompress.Compressors.Deflate
return _encoding.GetString(buffer, 0, buffer.Length);
}
private async Task<int> ReadAndValidateGzipHeaderAsync()
private async Task<int> ReadAndValidateGzipHeaderAsync(CancellationToken cancellationToken)
{
var totalBytesRead = 0;
// read the header on the first read
using var rented = MemoryPool<byte>.Shared.Rent(10);
int n = await _stream.ReadAsync(rented.Memory.Slice(0,10));
int n = await _stream.ReadAsync(rented.Memory.Slice(0,10), cancellationToken);
var header = rented.Memory;
// workitem 8501: handle edge case (decompress empty stream)
@@ -439,12 +449,12 @@ namespace SharpCompress.Compressors.Deflate
if ((header.Span[3] & 0x04) == 0x04)
{
// read and discard extra field
n = _stream.Read(header.Span.Slice(0, 2)); // 2-byte length field
n = await _stream.ReadAsync(header.Slice(0, 2), cancellationToken); // 2-byte length field
totalBytesRead += n;
var extraLength = (short)(header.Span[0] + header.Span[1] * 256);
using var extra = MemoryPool<byte>.Shared.Rent(extraLength);
n = await _stream.ReadAsync(extra.Memory.Slice(0, extraLength));
n = await _stream.ReadAsync(extra.Memory.Slice(0, extraLength), cancellationToken);
if (n != extraLength)
{
throw new ZlibException("Unexpected end-of-file reading GZIP header.");
@@ -461,7 +471,7 @@ namespace SharpCompress.Compressors.Deflate
}
if ((header.Span[3] & 0x02) == 0x02)
{
await ReadAsync(_buf1, 0, 1); // CRC16, ignore
await ReadAsync(_buf1, 0, 1, cancellationToken); // CRC16, ignore
}
return totalBytesRead;
@@ -490,7 +500,7 @@ namespace SharpCompress.Compressors.Deflate
z.AvailableBytesIn = 0;
if (_flavor == ZlibStreamFlavor.GZIP)
{
_gzipHeaderByteCount = await ReadAndValidateGzipHeaderAsync();
_gzipHeaderByteCount = await ReadAndValidateGzipHeaderAsync(cancellationToken);
// workitem 8501: handle edge case (decompress empty stream)
if (_gzipHeaderByteCount == 0)

14
src/SharpCompress/IO/CountingWritableSubStream.cs
View File

@@ -1,5 +1,7 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace SharpCompress.IO
{
@@ -41,6 +43,18 @@ namespace SharpCompress.IO
throw new NotSupportedException();
}
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
await Stream.WriteAsync(buffer, offset, count, cancellationToken);
Count += (uint)buffer.Length;
}
public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
await Stream.WriteAsync(buffer, cancellationToken);
Count += (uint)buffer.Length;
}
public override void Write(byte[] buffer, int offset, int count)
{
Stream.Write(buffer, offset, count);

14
src/SharpCompress/IO/NonDisposingStream.cs
View File

@@ -55,11 +55,21 @@ namespace SharpCompress.IO
throw new NotImplementedException();
}
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = new CancellationToken())
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken)
{
return Stream.ReadAsync(buffer, cancellationToken);
}
public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state)
{
throw new NotImplementedException();
}
public override int EndRead(IAsyncResult asyncResult)
{
throw new NotImplementedException();
}
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
return Stream.ReadAsync(buffer, offset, count, cancellationToken);
@@ -82,7 +92,7 @@ namespace SharpCompress.IO
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
throw new NotImplementedException();
return Stream.WriteAsync(buffer, offset, count, cancellationToken);
}
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = new CancellationToken())

37
src/SharpCompress/IO/RewindableStream.cs
View File

@@ -115,6 +115,43 @@ namespace SharpCompress.IO
throw new NotImplementedException();
}
public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
//don't actually read if we don't really want to read anything
//currently a network stream bug on Windows for .NET Core
if (count == 0)
{
return 0;
}
int read;
if (isRewound && bufferStream.Position != bufferStream.Length)
{
read = await bufferStream.ReadAsync(buffer, offset, count, cancellationToken);
if (read < count)
{
int tempRead = await stream.ReadAsync(buffer, read, count - read, cancellationToken);
if (IsRecording)
{
await bufferStream.WriteAsync(buffer, read, tempRead, cancellationToken);
}
read += tempRead;
}
if (bufferStream.Position == bufferStream.Length && !IsRecording)
{
isRewound = false;
bufferStream.SetLength(0);
}
return read;
}
read = await stream.ReadAsync(buffer, cancellationToken);
if (IsRecording)
{
await bufferStream.WriteAsync(buffer, cancellationToken);
}
return read;
}
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
var count = buffer.Length;

12
src/SharpCompress/Readers/Zip/ZipReader.cs
View File

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.IO;
namespace SharpCompress.Readers.Zip
{
@@ -40,7 +41,16 @@ namespace SharpCompress.Readers.Zip
protected override async IAsyncEnumerable<ZipEntry> GetEntries(Stream stream, [EnumeratorCancellation] CancellationToken cancellationToken)
{
await foreach (ZipHeader h in _headerFactory.ReadStreamHeader(stream, cancellationToken).WithCancellation(cancellationToken))
RewindableStream rewindableStream;
if (stream is RewindableStream rs)
{
rewindableStream = rs;
}
else
{
rewindableStream = new RewindableStream(stream);
}
await foreach (ZipHeader h in _headerFactory.ReadStreamHeader(rewindableStream, cancellationToken).WithCancellation(cancellationToken))
{
if (h != null)
{

50
src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs
View File

@@ -2,6 +2,8 @@
using System.Buffers.Binary;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
@@ -30,7 +32,7 @@ namespace SharpCompress.Writers.Zip
internal ushort Zip64HeaderOffset { get; set; }
internal ulong HeaderOffset { get; }
internal uint Write(Stream outputStream)
internal async ValueTask<uint> WriteAsync(Stream outputStream, CancellationToken cancellationToken)
{
byte[] encodedFilename = archiveEncoding.Encode(fileName);
byte[] encodedComment = archiveEncoding.Encode(Comment ?? string.Empty);
@@ -73,61 +75,61 @@ namespace SharpCompress.Writers.Zip
byte[] intBuf = new byte[] { 80, 75, 1, 2, version, 0, version, 0 };
//constant sig, then version made by, then version to extract
outputStream.Write(intBuf, 0, 8);
await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken);
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags);
outputStream.Write(intBuf, 0, 2);
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken);
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)usedCompression);
outputStream.Write(intBuf, 0, 2); // zipping method
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // zipping method
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, ModificationTime.DateTimeToDosTime());
outputStream.Write(intBuf, 0, 4);
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken);
// zipping date and time
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, Crc);
outputStream.Write(intBuf, 0, 4); // file CRC
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // file CRC
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, compressedvalue);
outputStream.Write(intBuf, 0, 4); // compressed file size
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // compressed file size
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, decompressedvalue);
outputStream.Write(intBuf, 0, 4); // uncompressed file size
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // uncompressed file size
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)encodedFilename.Length);
outputStream.Write(intBuf, 0, 2); // Filename in zip
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // Filename in zip
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)extralength);
outputStream.Write(intBuf, 0, 2); // extra length
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // extra length
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)encodedComment.Length);
outputStream.Write(intBuf, 0, 2);
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken);
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0);
outputStream.Write(intBuf, 0, 2); // disk=0
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // disk=0
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags);
outputStream.Write(intBuf, 0, 2); // file type: binary
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // file type: binary
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)flags);
outputStream.Write(intBuf, 0, 2); // Internal file attributes
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken); // Internal file attributes
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0x8100);
outputStream.Write(intBuf, 0, 2);
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken);
// External file attributes (normal/readable)
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, headeroffsetvalue);
outputStream.Write(intBuf, 0, 4); // Offset of header
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // Offset of header
outputStream.Write(encodedFilename, 0, encodedFilename.Length);
await outputStream.WriteAsync(encodedFilename, 0, encodedFilename.Length, cancellationToken);
if (zip64)
{
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0x0001);
outputStream.Write(intBuf, 0, 2);
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken);
BinaryPrimitives.WriteUInt16LittleEndian(intBuf, (ushort)(extralength - 4));
outputStream.Write(intBuf, 0, 2);
await outputStream.WriteAsync(intBuf, 0, 2, cancellationToken);
BinaryPrimitives.WriteUInt64LittleEndian(intBuf, Decompressed);
outputStream.Write(intBuf, 0, 8);
await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken);
BinaryPrimitives.WriteUInt64LittleEndian(intBuf, Compressed);
outputStream.Write(intBuf, 0, 8);
await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken);
BinaryPrimitives.WriteUInt64LittleEndian(intBuf, HeaderOffset);
outputStream.Write(intBuf, 0, 8);
await outputStream.WriteAsync(intBuf, 0, 8, cancellationToken);
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, 0);
outputStream.Write(intBuf, 0, 4); // VolumeNumber = 0
await outputStream.WriteAsync(intBuf, 0, 4, cancellationToken); // VolumeNumber = 0
}
outputStream.Write(encodedComment, 0, encodedComment.Length);
await outputStream.WriteAsync(encodedComment, 0, encodedComment.Length, cancellationToken);
return (uint)(8 + 2 + 2 + 4 + 4 + 4 + 4 + 2 + 2 + 2
+ 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + extralength + encodedComment.Length);

4
src/SharpCompress/Writers/Zip/ZipWriter.cs
View File

@@ -22,7 +22,7 @@ namespace SharpCompress.Writers.Zip
{
private readonly CompressionType compressionType;
private readonly CompressionLevel compressionLevel;
private readonly List<ZipCentralDirectoryEntry> entries = new List<ZipCentralDirectoryEntry>();
private readonly List<ZipCentralDirectoryEntry> entries = new();
private readonly string zipComment;
private long streamPosition;
private PpmdProperties? ppmdProps;
@@ -61,7 +61,7 @@ namespace SharpCompress.Writers.Zip
ulong size = 0;
foreach (ZipCentralDirectoryEntry entry in entries)
{
size += entry.Write(OutputStream);
size += await entry.WriteAsync(OutputStream, CancellationToken.None);
}
await WriteEndRecordAsync(size);
await base.DisposeAsyncCore();