starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-22 23:15:20 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Merge pull request #211 from kenkendk/add_zip64

Browse Source 
Add zip64
This commit is contained in:
Adam Hathcock
2017-03-13 10:23:26 +00:00
committed by GitHub
parent b3a4fed8be 726b9c80f6
commit f8538403e4
8 changed files with 500 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs
View File

@@ -63,6 +63,10 @@ namespace SharpCompress.Common.Zip.Headers
internal override void Write(BinaryWriter writer)
{
var zip64 = CompressedSize >= uint.MaxValue || UncompressedSize >= uint.MaxValue || RelativeOffsetOfEntryHeader >= uint.MaxValue;
if (zip64)
Version = (ushort)(Version > 45 ? Version : 45);
writer.Write(Version);
writer.Write(VersionNeededToExtract);
writer.Write((ushort)Flags);
@@ -70,24 +74,40 @@ namespace SharpCompress.Common.Zip.Headers
writer.Write(LastModifiedTime);
writer.Write(LastModifiedDate);
writer.Write(Crc);
writer.Write((uint)CompressedSize);
writer.Write((uint)UncompressedSize);
writer.Write(zip64 ? uint.MaxValue : CompressedSize);
writer.Write(zip64 ? uint.MaxValue : UncompressedSize);
byte[] nameBytes = EncodeString(Name);
writer.Write((ushort)nameBytes.Length);
//writer.Write((ushort)Extra.Length);
writer.Write((ushort)0);
if (zip64)
{
writer.Write((ushort)(2 + 2 + 8 + 8 + 8 + 4));
}
else
{
//writer.Write((ushort)Extra.Length);
writer.Write((ushort)0);
}
writer.Write((ushort)Comment.Length);
writer.Write(DiskNumberStart);
writer.Write(InternalFileAttributes);
writer.Write(ExternalFileAttributes);
writer.Write(RelativeOffsetOfEntryHeader);
writer.Write(zip64 ? uint.MaxValue : RelativeOffsetOfEntryHeader);
writer.Write(nameBytes);
// writer.Write(Extra);
if (zip64)
{
writer.Write((ushort)0x0001);
writer.Write((ushort)((8 + 8 + 8 + 4)));
writer.Write((ulong)UncompressedSize);
writer.Write((ulong)CompressedSize);
writer.Write((ulong)RelativeOffsetOfEntryHeader);
writer.Write((uint)0); // VolumeNumber = 0
}
writer.Write(Comment);
}

33
src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs
View File

@@ -49,25 +49,52 @@ namespace SharpCompress.Common.Zip.Headers
internal override void Write(BinaryWriter writer)
{
if (IsZip64)
Version = (ushort)(Version > 45 ? Version : 45);
writer.Write(Version);
writer.Write((ushort)Flags);
writer.Write((ushort)CompressionMethod);
writer.Write(LastModifiedTime);
writer.Write(LastModifiedDate);
writer.Write(Crc);
writer.Write((uint)CompressedSize);
writer.Write((uint)UncompressedSize);
if (IsZip64)
{
writer.Write(uint.MaxValue);
writer.Write(uint.MaxValue);
}
else
{
writer.Write(CompressedSize);
writer.Write(UncompressedSize);
}
byte[] nameBytes = EncodeString(Name);
writer.Write((ushort)nameBytes.Length);
writer.Write((ushort)0);
if (IsZip64)
{
writer.Write((ushort)(2 + 2 + (2 * 8)));
}
else
{
writer.Write((ushort)0);
}
//if (Extra != null)
//{
// writer.Write(Extra);
//}
writer.Write(nameBytes);
if (IsZip64)
{
writer.Write((ushort)0x0001);
writer.Write((ushort)(2 * 8));
writer.Write((ulong)CompressedSize);
writer.Write((ulong)UncompressedSize);
}
}
internal ushort Version { get; private set; }

2
src/SharpCompress/IO/CountingWritableSubStream.cs
View File

@@ -12,7 +12,7 @@ namespace SharpCompress.IO
writableStream = stream;
}
public uint Count { get; private set; }
public ulong Count { get; private set; }
public override bool CanRead { get { return false; } }

53
src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs
View File

@@ -13,36 +13,56 @@ namespace SharpCompress.Writers.Zip
internal DateTime? ModificationTime { get; set; }
internal string Comment { get; set; }
internal uint Crc { get; set; }
internal uint HeaderOffset { get; set; }
internal uint Compressed { get; set; }
internal uint Decompressed { get; set; }
internal ulong HeaderOffset { get; set; }
internal ulong Compressed { get; set; }
internal ulong Decompressed { get; set; }
internal ushort Zip64HeaderOffset { get; set; }
internal uint Write(Stream outputStream, ZipCompressionMethod compression)
{
byte[] encodedFilename = Encoding.UTF8.GetBytes(FileName);
byte[] encodedComment = Encoding.UTF8.GetBytes(Comment);
//constant sig, then version made by, compabitility, then version to extract
outputStream.Write(new byte[] {80, 75, 1, 2, 0x14, 0, 0x0A, 0}, 0, 8);
var zip64_stream = Compressed >= uint.MaxValue || Decompressed >= uint.MaxValue;
var zip64 = zip64_stream || HeaderOffset >= uint.MaxValue || Zip64HeaderOffset != 0;
var compressedvalue = zip64 ? uint.MaxValue : (uint)Compressed;
var decompressedvalue = zip64 ? uint.MaxValue : (uint)Decompressed;
var headeroffsetvalue = zip64 ? uint.MaxValue : (uint)HeaderOffset;
var extralength = zip64 ? (2 + 2 + 8 + 8 + 8 + 4) : 0;
var version = (byte)(zip64 ? 45 : 10);
HeaderFlags flags = HeaderFlags.UTF8;
if (!outputStream.CanSeek)
{
flags |= HeaderFlags.UsePostDataDescriptor;
// Cannot use data descriptors with zip64:
// https://blogs.oracle.com/xuemingshen/entry/is_zipinput_outputstream_handling_of
// We check that streams are not written too large in the ZipWritingStream,
// so this extra guard is not required, but kept to simplify changing the code
// once the zip64 post-data issue is resolved
if (!zip64_stream)
flags |= HeaderFlags.UsePostDataDescriptor;
if (compression == ZipCompressionMethod.LZMA)
{
flags |= HeaderFlags.Bit1; // eos marker
}
}
//constant sig, then version made by, compabitility, then version to extract
outputStream.Write(new byte[] { 80, 75, 1, 2, 0x14, 0, version, 0 }, 0, 8);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)flags), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)compression), 0, 2); // zipping method
outputStream.Write(DataConverter.LittleEndian.GetBytes(ModificationTime.DateTimeToDosTime()), 0, 4);
// zipping date and time
outputStream.Write(DataConverter.LittleEndian.GetBytes(Crc), 0, 4); // file CRC
outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 4); // compressed file size
outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 4); // uncompressed file size
outputStream.Write(DataConverter.LittleEndian.GetBytes(compressedvalue), 0, 4); // compressed file size
outputStream.Write(DataConverter.LittleEndian.GetBytes(decompressedvalue), 0, 4); // uncompressed file size
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // Filename in zip
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // disk=0
@@ -51,13 +71,24 @@ namespace SharpCompress.Writers.Zip
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x8100), 0, 2);
// External file attributes (normal/readable)
outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 4); // Offset of header
outputStream.Write(DataConverter.LittleEndian.GetBytes(headeroffsetvalue), 0, 4); // Offset of header
outputStream.Write(encodedFilename, 0, encodedFilename.Length);
if (zip64)
{
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(extralength - 4)), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 8);
outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 8);
outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 8);
outputStream.Write(DataConverter.LittleEndian.GetBytes(0), 0, 4); // VolumeNumber = 0
}
outputStream.Write(encodedComment, 0, encodedComment.Length);
return (uint)(8 + 2 + 2 + 4 + 4 + 4 + 4 + 2 + 2 + 2
+ 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + encodedComment.Length);
+ 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + extralength + encodedComment.Length);
}
}
}

186
src/SharpCompress/Writers/Zip/ZipWriter.cs
View File

@@ -23,11 +23,13 @@ namespace SharpCompress.Writers.Zip
private readonly string zipComment;
private long streamPosition;
private PpmdProperties ppmdProps;
private bool isZip64;
public ZipWriter(Stream destination, ZipWriterOptions zipWriterOptions)
: base(ArchiveType.Zip)
{
zipComment = zipWriterOptions.ArchiveComment ?? string.Empty;
isZip64 = zipWriterOptions.UseZip64;
compressionType = zipWriterOptions.CompressionType;
compressionLevel = zipWriterOptions.DeflateCompressionLevel;
@@ -50,7 +52,7 @@ namespace SharpCompress.Writers.Zip
{
if (isDisposing)
{
uint size = 0;
ulong size = 0;
foreach (ZipCentralDirectoryEntry entry in entries)
{
size += entry.Write(OutputStream, ToZipCompressionMethod(compressionType));
@@ -114,10 +116,15 @@ namespace SharpCompress.Writers.Zip
Comment = options.EntryComment,
FileName = entryPath,
ModificationTime = options.ModificationDateTime,
HeaderOffset = (uint)streamPosition
};
HeaderOffset = (ulong)streamPosition
};
var headersize = (uint)WriteHeader(entryPath, options);
// Use the archive default setting for zip64 and allow overrides
var useZip64 = isZip64;
if (options.EnableZip64.HasValue)
useZip64 = options.EnableZip64.Value;
var headersize = (uint)WriteHeader(entryPath, options, entry, useZip64);
streamPosition += headersize;
return new ZipWritingStream(this, OutputStream, entry,
ToZipCompressionMethod(options.CompressionType ?? compressionType),
@@ -137,24 +144,32 @@ namespace SharpCompress.Writers.Zip
return filename.Trim('/');
}
private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions)
private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions, ZipCentralDirectoryEntry entry, bool useZip64)
{
// We err on the side of caution until the zip specification clarifies how to support this
if (!OutputStream.CanSeek && useZip64)
throw new NotSupportedException("Zip64 extensions are not supported on non-seekable streams");
var explicitZipCompressionInfo = ToZipCompressionMethod(zipWriterEntryOptions.CompressionType ?? compressionType);
byte[] encodedFilename = ArchiveEncoding.Default.GetBytes(filename);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.ENTRY_HEADER_BYTES), 0, 4);
if (explicitZipCompressionInfo == ZipCompressionMethod.Deflate)
{
OutputStream.Write(new byte[] {20, 0}, 0, 2); //older version which is more compatible
if (OutputStream.CanSeek && useZip64)
OutputStream.Write(new byte[] { 45, 0 }, 0, 2); //smallest allowed version for zip64
else
OutputStream.Write(new byte[] { 20, 0 }, 0, 2); //older version which is more compatible
}
else
{
OutputStream.Write(new byte[] {63, 0}, 0, 2); //version says we used PPMd or LZMA
OutputStream.Write(new byte[] { 63, 0 }, 0, 2); //version says we used PPMd or LZMA
}
HeaderFlags flags = ArchiveEncoding.Default == Encoding.UTF8 ? HeaderFlags.UTF8 : 0;
if (!OutputStream.CanSeek)
{
flags |= HeaderFlags.UsePostDataDescriptor;
if (explicitZipCompressionInfo == ZipCompressionMethod.LZMA)
{
flags |= HeaderFlags.Bit1; // eos marker
@@ -165,14 +180,25 @@ namespace SharpCompress.Writers.Zip
OutputStream.Write(DataConverter.LittleEndian.GetBytes(zipWriterEntryOptions.ModificationDateTime.DateTimeToDosTime()), 0, 4);
// zipping date and time
OutputStream.Write(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 0, 12);
OutputStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 12);
// unused CRC, un/compressed size, updated later
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // filename length
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length
var extralength = 0;
if (OutputStream.CanSeek && useZip64)
extralength = 2 + 2 + 8 + 8;
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length
OutputStream.Write(encodedFilename, 0, encodedFilename.Length);
return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length;
if (extralength != 0)
{
OutputStream.Write(new byte[extralength], 0, extralength); // reserve space for zip64 data
entry.Zip64HeaderOffset = (ushort)(6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length);
}
return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length + extralength;
}
private void WriteFooter(uint crc, uint compressed, uint uncompressed)
@@ -182,15 +208,58 @@ namespace SharpCompress.Writers.Zip
OutputStream.Write(DataConverter.LittleEndian.GetBytes(uncompressed), 0, 4);
}
private void WriteEndRecord(uint size)
private void WritePostdataDescriptor(uint crc, ulong compressed, ulong uncompressed)
{
OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(crc), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)compressed), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)uncompressed), 0, 4);
}
private void WriteEndRecord(ulong size)
{
byte[] encodedComment = ArchiveEncoding.Default.GetBytes(zipComment);
var zip64 = isZip64 || entries.Count > ushort.MaxValue || streamPosition >= uint.MaxValue || size >= uint.MaxValue;
var sizevalue = size >= uint.MaxValue ? uint.MaxValue : (uint)size;
var streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streamPosition;
if (zip64)
{
var recordlen = 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8;
// Write zip64 end of central directory record
OutputStream.Write(new byte[] { 80, 75, 6, 6 }, 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)recordlen), 0, 8); // Size of zip64 end of central directory record
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // Made by
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)45), 0, 2); // Version needed
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Disk number
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Central dir disk
// TODO: entries.Count is int, so max 2^31 files
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Entries in this disk
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Total entries
OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 8); // Central Directory size
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition), 0, 8); // Disk offset
// Write zip64 end of central directory locator
OutputStream.Write(new byte[] { 80, 75, 6, 7 }, 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(0uL), 0, 4); // Entry disk
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition + size), 0, 8); // Offset to the zip64 central directory
OutputStream.Write(DataConverter.LittleEndian.GetBytes(0u), 0, 4); // Number of disks
streamPosition += recordlen + (4 + 4 + 8 + 4);
streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streampositionvalue;
}
// Write normal end of central directory record
OutputStream.Write(new byte[] {80, 75, 5, 6, 0, 0, 0, 0}, 0, 8);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streamPosition), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(sizevalue), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streampositionvalue), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2);
OutputStream.Write(encodedComment, 0, encodedComment.Length);
}
@@ -207,7 +276,10 @@ namespace SharpCompress.Writers.Zip
private readonly ZipCompressionMethod zipCompressionMethod;
private readonly CompressionLevel compressionLevel;
private CountingWritableSubStream counting;
private uint decompressed;
private ulong decompressed;
// Flag to prevent throwing exceptions on Dispose
private bool limitsExceeded;
internal ZipWritingStream(ZipWriter writer, Stream originalStream, ZipCentralDirectoryEntry entry,
ZipCompressionMethod zipCompressionMethod, CompressionLevel compressionLevel)
@@ -280,23 +352,69 @@ namespace SharpCompress.Writers.Zip
if (disposing)
{
writeStream.Dispose();
if (limitsExceeded)
{
// We have written invalid data into the archive,
// so we destroy it now, instead of allowing the user to continue
// with a defunct archive
originalStream.Dispose();
return;
}
entry.Crc = (uint)crc.Crc32Result;
entry.Compressed = counting.Count;
entry.Decompressed = decompressed;
var zip64 = entry.Compressed >= uint.MaxValue || entry.Decompressed >= uint.MaxValue;
var compressedvalue = zip64 ? uint.MaxValue : (uint)counting.Count;
var decompressedvalue = zip64 ? uint.MaxValue : (uint)entry.Decompressed;
if (originalStream.CanSeek)
{
originalStream.Position = entry.HeaderOffset + 6;
originalStream.Position = (long)(entry.HeaderOffset + 6);
originalStream.WriteByte(0);
originalStream.Position = entry.HeaderOffset + 14;
writer.WriteFooter(entry.Crc, counting.Count, decompressed);
originalStream.Position = writer.streamPosition + entry.Compressed;
writer.streamPosition += entry.Compressed;
originalStream.Position = (long)(entry.HeaderOffset + 14);
writer.WriteFooter(entry.Crc, compressedvalue, decompressedvalue);
// Ideally, we should not throw from Dispose()
// We should not get here as the Write call checks the limits
if (zip64 && entry.Zip64HeaderOffset == 0)
throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
// If we have pre-allocated space for zip64 data,
// fill it out, even if it is not required
if (entry.Zip64HeaderOffset != 0)
{
originalStream.Position = (long)(entry.HeaderOffset + entry.Zip64HeaderOffset);
originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2);
originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(8 + 8)), 0, 2);
originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Decompressed), 0, 8);
originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Compressed), 0, 8);
}
originalStream.Position = writer.streamPosition + (long)entry.Compressed;
writer.streamPosition += (long)entry.Compressed;
}
else
{
originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
writer.WriteFooter(entry.Crc, counting.Count, decompressed);
writer.streamPosition += entry.Compressed + 16;
// We have a streaming archive, so we should add a post-data-descriptor,
// but we cannot as it does not hold the zip64 values
// Throwing an exception until the zip specification is clarified
// Ideally, we should not throw from Dispose()
// We should not get here as the Write call checks the limits
if (zip64)
throw new NotSupportedException("Streams larger than 4GiB are not supported for non-seekable streams");
originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
writer.WriteFooter(entry.Crc,
(uint)compressedvalue,
(uint)decompressedvalue);
writer.streamPosition += (long)entry.Compressed + 16;
}
writer.entries.Add(entry);
}
@@ -324,9 +442,33 @@ namespace SharpCompress.Writers.Zip
public override void Write(byte[] buffer, int offset, int count)
{
// We check the limits first, because we can keep the archive consistent
// if we can prevent the writes from happening
if (entry.Zip64HeaderOffset == 0)
{
// Pre-check, the counting.Count is not exact, as we do not know the size before having actually compressed it
if (limitsExceeded || ((decompressed + (uint)count) > uint.MaxValue) || (counting.Count + (uint)count) > uint.MaxValue)
throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
}
decompressed += (uint)count;
crc.SlurpBlock(buffer, offset, count);
writeStream.Write(buffer, offset, count);
if (entry.Zip64HeaderOffset == 0)
{
// Post-check, this is accurate
if ((decompressed > uint.MaxValue) || counting.Count > uint.MaxValue)
{
// We have written the data, so the archive is now broken
// Throwing the exception here, allows us to avoid
// throwing an exception in Dispose() which is discouraged
// as it can mask other errors
limitsExceeded = true;
throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
}
}
}
}

8
src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs
View File

@@ -15,5 +15,13 @@ namespace SharpCompress.Writers.Zip
public string EntryComment { get; set; }
public DateTime? ModificationDateTime { get; set; }
/// <summary>
/// Allocate an extra 20 bytes for this entry to store,
/// 64 bit length values, thus enabling streams
/// larger than 4GiB.
/// This option is not supported with non-seekable streams.
/// </summary>
public bool? EnableZip64 { get; set; }
}
}

11
src/SharpCompress/Writers/Zip/ZipWriterOptions.cs
View File

@@ -15,6 +15,8 @@ namespace SharpCompress.Writers.Zip
: base(options.CompressionType)
{
LeaveStreamOpen = options.LeaveStreamOpen;
if (options is ZipWriterOptions)
UseZip64 = ((ZipWriterOptions)options).UseZip64;
}
/// <summary>
/// When CompressionType.Deflate is used, this property is referenced. Defaults to CompressionLevel.Default.
@@ -22,5 +24,14 @@ namespace SharpCompress.Writers.Zip
public CompressionLevel DeflateCompressionLevel { get; set; } = CompressionLevel.Default;
public string ArchiveComment { get; set; }
/// <summary>
/// Sets a value indicating if zip64 support is enabled.
/// If this is not set, individual stream lengths cannot exceed 4 GiB.
/// This option is not supported for non-seekable streams.
/// Archives larger than 4GiB are supported as long as all streams
/// are less than 4GiB in length.
/// </summary>
public bool UseZip64 { get; set; }
}
}

218
test/SharpCompress.Test/Zip/Zip64Tests.cs Normal file
View File

@@ -0,0 +1,218 @@
using System;
using System.IO;
using System.Linq;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Readers;
using SharpCompress.Readers.Zip;
using SharpCompress.Writers;
using SharpCompress.Writers.Zip;
using Xunit;
namespace SharpCompress.Test
{
public class Zip64Tests : WriterTests
{
public Zip64Tests()
: base(ArchiveType.Zip)
{
}
// 4GiB + 1
const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1;
[Fact]
public void Zip64_Single_Large_File()
{
// One single file, requires zip64
RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: false);
}
[Fact]
public void Zip64_Two_Large_Files()
{
// One single file, requires zip64
RunSingleTest(2, FOUR_GB_LIMIT, set_zip64: true, forward_only: false);
}
[Fact]
public void Zip64_Two_Small_files()
{
// Multiple files, does not require zip64
RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: false);
}
[Fact]
public void Zip64_Two_Small_files_stream()
{
// Multiple files, does not require zip64, and works with streams
RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: true);
}
[Fact]
public void Zip64_Two_Small_Files_Zip64()
{
// Multiple files, use zip64 even though it is not required
RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: true, forward_only: false);
}
[Fact]
public void Zip64_Single_Large_File_Fail()
{
try
{
// One single file, should fail
RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: false);
throw new Exception("Test did not fail?");
}
catch (NotSupportedException)
{
}
}
[Fact]
public void Zip64_Single_Large_File_Zip64_Streaming_Fail()
{
try
{
// One single file, should fail (fast) with zip64
RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: true);
throw new Exception("Test did not fail?");
}
catch (NotSupportedException)
{
}
}
[Fact]
public void Zip64_Single_Large_File_Streaming_Fail()
{
try
{
// One single file, should fail once the write discovers the problem
RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: true);
throw new Exception("Test did not fail?");
}
catch (NotSupportedException)
{
}
}
public void RunSingleTest(long files, long filesize, bool set_zip64, bool forward_only, long write_chunk_size = 1024 * 1024, string filename = "zip64-test.zip")
{
ResetScratch();
filename = Path.Combine(SCRATCH2_FILES_PATH, filename);
if (File.Exists(filename))
File.Delete(filename);
if (!File.Exists(filename))
CreateZipArchive(filename, files, filesize, write_chunk_size, set_zip64, forward_only);
var resForward = ReadForwardOnly(filename);
if (resForward.Item1 != files)
throw new Exception($"Incorrect number of items reported: {resForward.Item1}, should have been {files}");
if (resForward.Item2 != files * filesize)
throw new Exception($"Incorrect combined size reported: {resForward.Item2}, should have been {files * filesize}");
var resArchive = ReadArchive(filename);
if (resArchive.Item1 != files)
throw new Exception($"Incorrect number of items reported: {resArchive.Item1}, should have been {files}");
if (resArchive.Item2 != files * filesize)
throw new Exception($"Incorrect number of items reported: {resArchive.Item2}, should have been {files * filesize}");
}
public void CreateZipArchive(string filename, long files, long filesize, long chunksize, bool set_zip64, bool forward_only)
{
var data = new byte[chunksize];
// Use deflate for speed
var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = set_zip64 };
// Use no compression to ensure we hit the limits (actually inflates a bit, but seems better than using method==Store)
var eo = new ZipWriterEntryOptions() { DeflateCompressionLevel = SharpCompress.Compressors.Deflate.CompressionLevel.None };
using (var zip = File.OpenWrite(filename))
using(var st = forward_only ? (Stream)new NonSeekableStream(zip) : zip)
using (var zipWriter = (ZipWriter)WriterFactory.Open(st, ArchiveType.Zip, opts))
{
for (var i = 0; i < files; i++)
using (var str = zipWriter.WriteToStream(i.ToString(), eo))
{
var left = filesize;
while (left > 0)
{
var b = (int)Math.Min(left, data.Length);
str.Write(data, 0, b);
left -= b;
}
}
}
}
public Tuple<long, long> ReadForwardOnly(string filename)
{
long count = 0;
long size = 0;
Common.Zip.ZipEntry prev = null;
using (var fs = File.OpenRead(filename))
using (var rd = ZipReader.Open(fs, new ReaderOptions() { LookForHeader = false }))
while (rd.MoveToNextEntry())
{
using (rd.OpenEntryStream())
{ }
count++;
if (prev != null)
size += prev.Size;
prev = rd.Entry;
}
if (prev != null)
size += prev.Size;
return new Tuple<long, long>(count, size);
}
public Tuple<long, long> ReadArchive(string filename)
{
using (var archive = ArchiveFactory.Open(filename))
{
return new Tuple<long, long>(
archive.Entries.Count(),
archive.Entries.Select(x => x.Size).Sum()
);
}
}
/// <summary>
/// Helper to create non-seekable streams from filestream
/// </summary>
private class NonSeekableStream : Stream
{
private readonly Stream stream;
public NonSeekableStream(Stream s) { stream = s; }
public override bool CanRead { get { return stream.CanRead; } }
public override bool CanSeek { get { return false; } }
public override bool CanWrite { get { return stream.CanWrite; } }
public override long Length { get { throw new NotImplementedException(); } }
public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } }
public override void Flush() { stream.Flush(); }
public override int Read(byte[] buffer, int offset, int count)
{ return stream.Read(buffer, offset, count); }
public override long Seek(long offset, SeekOrigin origin)
{ throw new NotImplementedException(); }
public override void SetLength(long value)
{ throw new NotImplementedException(); }
public override void Write(byte[] buffer, int offset, int count)
{ stream.Write(buffer, offset, count); }
}
}
}