diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs
index 3b50d118..acca3bb1 100644
--- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs
+++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs
@@ -63,6 +63,10 @@ namespace SharpCompress.Common.Zip.Headers
internal override void Write(BinaryWriter writer)
{
+ var zip64 = CompressedSize >= uint.MaxValue || UncompressedSize >= uint.MaxValue || RelativeOffsetOfEntryHeader >= uint.MaxValue;
+ if (zip64)
+ Version = (ushort)(Version > 45 ? Version : 45);
+
writer.Write(Version);
writer.Write(VersionNeededToExtract);
writer.Write((ushort)Flags);
@@ -70,24 +74,40 @@ namespace SharpCompress.Common.Zip.Headers
writer.Write(LastModifiedTime);
writer.Write(LastModifiedDate);
writer.Write(Crc);
- writer.Write((uint)CompressedSize);
- writer.Write((uint)UncompressedSize);
+ writer.Write(zip64 ? uint.MaxValue : CompressedSize);
+ writer.Write(zip64 ? uint.MaxValue : UncompressedSize);
byte[] nameBytes = EncodeString(Name);
writer.Write((ushort)nameBytes.Length);
- //writer.Write((ushort)Extra.Length);
- writer.Write((ushort)0);
+ if (zip64)
+ {
+ writer.Write((ushort)(2 + 2 + 8 + 8 + 8 + 4));
+ }
+ else
+ {
+ //writer.Write((ushort)Extra.Length);
+ writer.Write((ushort)0);
+ }
writer.Write((ushort)Comment.Length);
writer.Write(DiskNumberStart);
writer.Write(InternalFileAttributes);
writer.Write(ExternalFileAttributes);
- writer.Write(RelativeOffsetOfEntryHeader);
+ writer.Write(zip64 ? uint.MaxValue : RelativeOffsetOfEntryHeader);
writer.Write(nameBytes);
- // writer.Write(Extra);
+ if (zip64)
+ {
+ writer.Write((ushort)0x0001);
+ writer.Write((ushort)((8 + 8 + 8 + 4)));
+
+ writer.Write((ulong)UncompressedSize);
+ writer.Write((ulong)CompressedSize);
+ writer.Write((ulong)RelativeOffsetOfEntryHeader);
+ writer.Write((uint)0); // VolumeNumber = 0
+ }
writer.Write(Comment);
}
diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs
index 2e311cc8..40e2a662 100644
--- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs
+++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs
@@ -49,25 +49,52 @@ namespace SharpCompress.Common.Zip.Headers
internal override void Write(BinaryWriter writer)
{
+ if (IsZip64)
+ Version = (ushort)(Version > 45 ? Version : 45);
+
writer.Write(Version);
+
writer.Write((ushort)Flags);
writer.Write((ushort)CompressionMethod);
writer.Write(LastModifiedTime);
writer.Write(LastModifiedDate);
writer.Write(Crc);
- writer.Write((uint)CompressedSize);
- writer.Write((uint)UncompressedSize);
+
+ if (IsZip64)
+ {
+ writer.Write(uint.MaxValue);
+ writer.Write(uint.MaxValue);
+ }
+ else
+ {
+ writer.Write(CompressedSize);
+ writer.Write(UncompressedSize);
+ }
byte[] nameBytes = EncodeString(Name);
writer.Write((ushort)nameBytes.Length);
- writer.Write((ushort)0);
+ if (IsZip64)
+ {
+ writer.Write((ushort)(2 + 2 + (2 * 8)));
+ }
+ else
+ {
+ writer.Write((ushort)0);
+ }
//if (Extra != null)
//{
// writer.Write(Extra);
//}
writer.Write(nameBytes);
+ if (IsZip64)
+ {
+ writer.Write((ushort)0x0001);
+ writer.Write((ushort)(2 * 8));
+ writer.Write((ulong)CompressedSize);
+ writer.Write((ulong)UncompressedSize);
+ }
}
internal ushort Version { get; private set; }
diff --git a/src/SharpCompress/IO/CountingWritableSubStream.cs b/src/SharpCompress/IO/CountingWritableSubStream.cs
index 17313612..51989b75 100644
--- a/src/SharpCompress/IO/CountingWritableSubStream.cs
+++ b/src/SharpCompress/IO/CountingWritableSubStream.cs
@@ -12,7 +12,7 @@ namespace SharpCompress.IO
writableStream = stream;
}
- public uint Count { get; private set; }
+ public ulong Count { get; private set; }
public override bool CanRead { get { return false; } }
diff --git a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs
index 28b9aa63..3e29a0f5 100644
--- a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs
+++ b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs
@@ -13,36 +13,56 @@ namespace SharpCompress.Writers.Zip
internal DateTime? ModificationTime { get; set; }
internal string Comment { get; set; }
internal uint Crc { get; set; }
- internal uint HeaderOffset { get; set; }
- internal uint Compressed { get; set; }
- internal uint Decompressed { get; set; }
+ internal ulong HeaderOffset { get; set; }
+ internal ulong Compressed { get; set; }
+ internal ulong Decompressed { get; set; }
+ internal ushort Zip64HeaderOffset { get; set; }
internal uint Write(Stream outputStream, ZipCompressionMethod compression)
{
byte[] encodedFilename = Encoding.UTF8.GetBytes(FileName);
byte[] encodedComment = Encoding.UTF8.GetBytes(Comment);
- //constant sig, then version made by, compabitility, then version to extract
- outputStream.Write(new byte[] {80, 75, 1, 2, 0x14, 0, 0x0A, 0}, 0, 8);
+ var zip64_stream = Compressed >= uint.MaxValue || Decompressed >= uint.MaxValue;
+ var zip64 = zip64_stream || HeaderOffset >= uint.MaxValue || Zip64HeaderOffset != 0;
+
+ var compressedvalue = zip64 ? uint.MaxValue : (uint)Compressed;
+ var decompressedvalue = zip64 ? uint.MaxValue : (uint)Decompressed;
+ var headeroffsetvalue = zip64 ? uint.MaxValue : (uint)HeaderOffset;
+ var extralength = zip64 ? (2 + 2 + 8 + 8 + 8 + 4) : 0;
+ var version = (byte)(zip64 ? 45 : 10);
+
HeaderFlags flags = HeaderFlags.UTF8;
if (!outputStream.CanSeek)
{
- flags |= HeaderFlags.UsePostDataDescriptor;
+ // Cannot use data descriptors with zip64:
+ // https://blogs.oracle.com/xuemingshen/entry/is_zipinput_outputstream_handling_of
+
+ // We check that streams are not written too large in the ZipWritingStream,
+ // so this extra guard is not required, but kept to simplify changing the code
+ // once the zip64 post-data issue is resolved
+ if (!zip64_stream)
+ flags |= HeaderFlags.UsePostDataDescriptor;
+
if (compression == ZipCompressionMethod.LZMA)
{
flags |= HeaderFlags.Bit1; // eos marker
}
}
+
+ //constant sig, then version made by, compabitility, then version to extract
+ outputStream.Write(new byte[] { 80, 75, 1, 2, 0x14, 0, version, 0 }, 0, 8);
+
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)flags), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)compression), 0, 2); // zipping method
outputStream.Write(DataConverter.LittleEndian.GetBytes(ModificationTime.DateTimeToDosTime()), 0, 4);
// zipping date and time
outputStream.Write(DataConverter.LittleEndian.GetBytes(Crc), 0, 4); // file CRC
- outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 4); // compressed file size
- outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 4); // uncompressed file size
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(compressedvalue), 0, 4); // compressed file size
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(decompressedvalue), 0, 4); // uncompressed file size
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // Filename in zip
- outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length
+ outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2);
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // disk=0
@@ -51,13 +71,24 @@ namespace SharpCompress.Writers.Zip
outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x8100), 0, 2);
// External file attributes (normal/readable)
- outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 4); // Offset of header
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(headeroffsetvalue), 0, 4); // Offset of header
outputStream.Write(encodedFilename, 0, encodedFilename.Length);
+ if (zip64)
+ {
+ outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2);
+ outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(extralength - 4)), 0, 2);
+
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 8);
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 8);
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 8);
+ outputStream.Write(DataConverter.LittleEndian.GetBytes(0), 0, 4); // VolumeNumber = 0
+ }
+
outputStream.Write(encodedComment, 0, encodedComment.Length);
return (uint)(8 + 2 + 2 + 4 + 4 + 4 + 4 + 2 + 2 + 2
- + 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + encodedComment.Length);
+ + 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + extralength + encodedComment.Length);
}
}
}
\ No newline at end of file
diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs
index d89294b5..e1a001b5 100644
--- a/src/SharpCompress/Writers/Zip/ZipWriter.cs
+++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs
@@ -23,11 +23,13 @@ namespace SharpCompress.Writers.Zip
private readonly string zipComment;
private long streamPosition;
private PpmdProperties ppmdProps;
+ private bool isZip64;
public ZipWriter(Stream destination, ZipWriterOptions zipWriterOptions)
: base(ArchiveType.Zip)
{
zipComment = zipWriterOptions.ArchiveComment ?? string.Empty;
+ isZip64 = zipWriterOptions.UseZip64;
compressionType = zipWriterOptions.CompressionType;
compressionLevel = zipWriterOptions.DeflateCompressionLevel;
@@ -50,7 +52,7 @@ namespace SharpCompress.Writers.Zip
{
if (isDisposing)
{
- uint size = 0;
+ ulong size = 0;
foreach (ZipCentralDirectoryEntry entry in entries)
{
size += entry.Write(OutputStream, ToZipCompressionMethod(compressionType));
@@ -114,10 +116,15 @@ namespace SharpCompress.Writers.Zip
Comment = options.EntryComment,
FileName = entryPath,
ModificationTime = options.ModificationDateTime,
- HeaderOffset = (uint)streamPosition
- };
+ HeaderOffset = (ulong)streamPosition
+ };
- var headersize = (uint)WriteHeader(entryPath, options);
+ // Use the archive default setting for zip64 and allow overrides
+ var useZip64 = isZip64;
+ if (options.EnableZip64.HasValue)
+ useZip64 = options.EnableZip64.Value;
+
+ var headersize = (uint)WriteHeader(entryPath, options, entry, useZip64);
streamPosition += headersize;
return new ZipWritingStream(this, OutputStream, entry,
ToZipCompressionMethod(options.CompressionType ?? compressionType),
@@ -137,24 +144,32 @@ namespace SharpCompress.Writers.Zip
return filename.Trim('/');
}
- private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions)
+ private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions, ZipCentralDirectoryEntry entry, bool useZip64)
{
+ // We err on the side of caution until the zip specification clarifies how to support this
+ if (!OutputStream.CanSeek && useZip64)
+ throw new NotSupportedException("Zip64 extensions are not supported on non-seekable streams");
+
var explicitZipCompressionInfo = ToZipCompressionMethod(zipWriterEntryOptions.CompressionType ?? compressionType);
byte[] encodedFilename = ArchiveEncoding.Default.GetBytes(filename);
OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.ENTRY_HEADER_BYTES), 0, 4);
if (explicitZipCompressionInfo == ZipCompressionMethod.Deflate)
{
- OutputStream.Write(new byte[] {20, 0}, 0, 2); //older version which is more compatible
+ if (OutputStream.CanSeek && useZip64)
+ OutputStream.Write(new byte[] { 45, 0 }, 0, 2); //smallest allowed version for zip64
+ else
+ OutputStream.Write(new byte[] { 20, 0 }, 0, 2); //older version which is more compatible
}
else
{
- OutputStream.Write(new byte[] {63, 0}, 0, 2); //version says we used PPMd or LZMA
+ OutputStream.Write(new byte[] { 63, 0 }, 0, 2); //version says we used PPMd or LZMA
}
HeaderFlags flags = ArchiveEncoding.Default == Encoding.UTF8 ? HeaderFlags.UTF8 : 0;
if (!OutputStream.CanSeek)
{
flags |= HeaderFlags.UsePostDataDescriptor;
+
if (explicitZipCompressionInfo == ZipCompressionMethod.LZMA)
{
flags |= HeaderFlags.Bit1; // eos marker
@@ -165,14 +180,25 @@ namespace SharpCompress.Writers.Zip
OutputStream.Write(DataConverter.LittleEndian.GetBytes(zipWriterEntryOptions.ModificationDateTime.DateTimeToDosTime()), 0, 4);
// zipping date and time
- OutputStream.Write(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 0, 12);
+ OutputStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 12);
// unused CRC, un/compressed size, updated later
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // filename length
- OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length
+
+ var extralength = 0;
+ if (OutputStream.CanSeek && useZip64)
+ extralength = 2 + 2 + 8 + 8;
+
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length
OutputStream.Write(encodedFilename, 0, encodedFilename.Length);
- return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length;
+ if (extralength != 0)
+ {
+ OutputStream.Write(new byte[extralength], 0, extralength); // reserve space for zip64 data
+ entry.Zip64HeaderOffset = (ushort)(6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length);
+ }
+
+ return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length + extralength;
}
private void WriteFooter(uint crc, uint compressed, uint uncompressed)
@@ -182,15 +208,58 @@ namespace SharpCompress.Writers.Zip
OutputStream.Write(DataConverter.LittleEndian.GetBytes(uncompressed), 0, 4);
}
- private void WriteEndRecord(uint size)
+ private void WritePostdataDescriptor(uint crc, ulong compressed, ulong uncompressed)
+ {
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(crc), 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)compressed), 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)uncompressed), 0, 4);
+ }
+
+ private void WriteEndRecord(ulong size)
{
byte[] encodedComment = ArchiveEncoding.Default.GetBytes(zipComment);
+ var zip64 = isZip64 || entries.Count > ushort.MaxValue || streamPosition >= uint.MaxValue || size >= uint.MaxValue;
+ var sizevalue = size >= uint.MaxValue ? uint.MaxValue : (uint)size;
+ var streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streamPosition;
+
+ if (zip64)
+ {
+ var recordlen = 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8;
+
+ // Write zip64 end of central directory record
+ OutputStream.Write(new byte[] { 80, 75, 6, 6 }, 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)recordlen), 0, 8); // Size of zip64 end of central directory record
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // Made by
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)45), 0, 2); // Version needed
+
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Disk number
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Central dir disk
+
+ // TODO: entries.Count is int, so max 2^31 files
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Entries in this disk
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Total entries
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 8); // Central Directory size
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition), 0, 8); // Disk offset
+
+ // Write zip64 end of central directory locator
+ OutputStream.Write(new byte[] { 80, 75, 6, 7 }, 0, 4);
+
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(0uL), 0, 4); // Entry disk
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition + size), 0, 8); // Offset to the zip64 central directory
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(0u), 0, 4); // Number of disks
+
+ streamPosition += recordlen + (4 + 4 + 8 + 4);
+ streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streampositionvalue;
+ }
+
+ // Write normal end of central directory record
OutputStream.Write(new byte[] {80, 75, 5, 6, 0, 0, 0, 0}, 0, 8);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2);
- OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 4);
- OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streamPosition), 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes(sizevalue), 0, 4);
+ OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streampositionvalue), 0, 4);
OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2);
OutputStream.Write(encodedComment, 0, encodedComment.Length);
}
@@ -207,7 +276,10 @@ namespace SharpCompress.Writers.Zip
private readonly ZipCompressionMethod zipCompressionMethod;
private readonly CompressionLevel compressionLevel;
private CountingWritableSubStream counting;
- private uint decompressed;
+ private ulong decompressed;
+
+ // Flag to prevent throwing exceptions on Dispose
+ private bool limitsExceeded;
internal ZipWritingStream(ZipWriter writer, Stream originalStream, ZipCentralDirectoryEntry entry,
ZipCompressionMethod zipCompressionMethod, CompressionLevel compressionLevel)
@@ -280,23 +352,69 @@ namespace SharpCompress.Writers.Zip
if (disposing)
{
writeStream.Dispose();
+
+ if (limitsExceeded)
+ {
+ // We have written invalid data into the archive,
+ // so we destroy it now, instead of allowing the user to continue
+ // with a defunct archive
+ originalStream.Dispose();
+ return;
+ }
+
entry.Crc = (uint)crc.Crc32Result;
entry.Compressed = counting.Count;
entry.Decompressed = decompressed;
+
+ var zip64 = entry.Compressed >= uint.MaxValue || entry.Decompressed >= uint.MaxValue;
+ var compressedvalue = zip64 ? uint.MaxValue : (uint)counting.Count;
+ var decompressedvalue = zip64 ? uint.MaxValue : (uint)entry.Decompressed;
+
if (originalStream.CanSeek)
{
- originalStream.Position = entry.HeaderOffset + 6;
+ originalStream.Position = (long)(entry.HeaderOffset + 6);
originalStream.WriteByte(0);
- originalStream.Position = entry.HeaderOffset + 14;
- writer.WriteFooter(entry.Crc, counting.Count, decompressed);
- originalStream.Position = writer.streamPosition + entry.Compressed;
- writer.streamPosition += entry.Compressed;
+
+ originalStream.Position = (long)(entry.HeaderOffset + 14);
+
+ writer.WriteFooter(entry.Crc, compressedvalue, decompressedvalue);
+
+ // Ideally, we should not throw from Dispose()
+ // We should not get here as the Write call checks the limits
+ if (zip64 && entry.Zip64HeaderOffset == 0)
+ throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
+
+ // If we have pre-allocated space for zip64 data,
+ // fill it out, even if it is not required
+ if (entry.Zip64HeaderOffset != 0)
+ {
+ originalStream.Position = (long)(entry.HeaderOffset + entry.Zip64HeaderOffset);
+ originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2);
+ originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(8 + 8)), 0, 2);
+
+ originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Decompressed), 0, 8);
+ originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Compressed), 0, 8);
+ }
+
+ originalStream.Position = writer.streamPosition + (long)entry.Compressed;
+ writer.streamPosition += (long)entry.Compressed;
}
else
{
- originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
- writer.WriteFooter(entry.Crc, counting.Count, decompressed);
- writer.streamPosition += entry.Compressed + 16;
+ // We have a streaming archive, so we should add a post-data-descriptor,
+ // but we cannot as it does not hold the zip64 values
+ // Throwing an exception until the zip specification is clarified
+
+ // Ideally, we should not throw from Dispose()
+ // We should not get here as the Write call checks the limits
+ if (zip64)
+ throw new NotSupportedException("Streams larger than 4GiB are not supported for non-seekable streams");
+
+ originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4);
+ writer.WriteFooter(entry.Crc,
+ (uint)compressedvalue,
+ (uint)decompressedvalue);
+ writer.streamPosition += (long)entry.Compressed + 16;
}
writer.entries.Add(entry);
}
@@ -324,9 +442,33 @@ namespace SharpCompress.Writers.Zip
public override void Write(byte[] buffer, int offset, int count)
{
+ // We check the limits first, because we can keep the archive consistent
+ // if we can prevent the writes from happening
+ if (entry.Zip64HeaderOffset == 0)
+ {
+ // Pre-check, the counting.Count is not exact, as we do not know the size before having actually compressed it
+ if (limitsExceeded || ((decompressed + (uint)count) > uint.MaxValue) || (counting.Count + (uint)count) > uint.MaxValue)
+ throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
+ }
+
decompressed += (uint)count;
crc.SlurpBlock(buffer, offset, count);
writeStream.Write(buffer, offset, count);
+
+ if (entry.Zip64HeaderOffset == 0)
+ {
+ // Post-check, this is accurate
+ if ((decompressed > uint.MaxValue) || counting.Count > uint.MaxValue)
+ {
+ // We have written the data, so the archive is now broken
+ // Throwing the exception here, allows us to avoid
+ // throwing an exception in Dispose() which is discouraged
+ // as it can mask other errors
+ limitsExceeded = true;
+ throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option");
+ }
+ }
+
}
}
diff --git a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs
index 81f1d1c0..5f1d8152 100644
--- a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs
+++ b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs
@@ -15,5 +15,13 @@ namespace SharpCompress.Writers.Zip
public string EntryComment { get; set; }
public DateTime? ModificationDateTime { get; set; }
+
+ ///
+ /// Allocate an extra 20 bytes for this entry to store,
+ /// 64 bit length values, thus enabling streams
+ /// larger than 4GiB.
+ /// This option is not supported with non-seekable streams.
+ ///
+ public bool? EnableZip64 { get; set; }
}
}
\ No newline at end of file
diff --git a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs
index 662a50d4..81c2afbe 100644
--- a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs
+++ b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs
@@ -15,6 +15,8 @@ namespace SharpCompress.Writers.Zip
: base(options.CompressionType)
{
LeaveStreamOpen = options.LeaveStreamOpen;
+ if (options is ZipWriterOptions)
+ UseZip64 = ((ZipWriterOptions)options).UseZip64;
}
///
/// When CompressionType.Deflate is used, this property is referenced. Defaults to CompressionLevel.Default.
@@ -22,5 +24,14 @@ namespace SharpCompress.Writers.Zip
public CompressionLevel DeflateCompressionLevel { get; set; } = CompressionLevel.Default;
public string ArchiveComment { get; set; }
+
+ ///
+ /// Sets a value indicating if zip64 support is enabled.
+ /// If this is not set, individual stream lengths cannot exceed 4 GiB.
+ /// This option is not supported for non-seekable streams.
+ /// Archives larger than 4GiB are supported as long as all streams
+ /// are less than 4GiB in length.
+ ///
+ public bool UseZip64 { get; set; }
}
}
\ No newline at end of file
diff --git a/test/SharpCompress.Test/Zip/Zip64Tests.cs b/test/SharpCompress.Test/Zip/Zip64Tests.cs
new file mode 100644
index 00000000..4a4bf639
--- /dev/null
+++ b/test/SharpCompress.Test/Zip/Zip64Tests.cs
@@ -0,0 +1,218 @@
+using System;
+using System.IO;
+using System.Linq;
+using SharpCompress.Archives;
+using SharpCompress.Common;
+using SharpCompress.Readers;
+using SharpCompress.Readers.Zip;
+using SharpCompress.Writers;
+using SharpCompress.Writers.Zip;
+using Xunit;
+
+namespace SharpCompress.Test
+{
+ public class Zip64Tests : WriterTests
+ {
+ public Zip64Tests()
+ : base(ArchiveType.Zip)
+ {
+ }
+
+ // 4GiB + 1
+ const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1;
+
+ [Fact]
+ public void Zip64_Single_Large_File()
+ {
+ // One single file, requires zip64
+ RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: false);
+ }
+
+ [Fact]
+ public void Zip64_Two_Large_Files()
+ {
+ // One single file, requires zip64
+ RunSingleTest(2, FOUR_GB_LIMIT, set_zip64: true, forward_only: false);
+ }
+
+ [Fact]
+ public void Zip64_Two_Small_files()
+ {
+ // Multiple files, does not require zip64
+ RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: false);
+ }
+
+ [Fact]
+ public void Zip64_Two_Small_files_stream()
+ {
+ // Multiple files, does not require zip64, and works with streams
+ RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: true);
+ }
+
+ [Fact]
+ public void Zip64_Two_Small_Files_Zip64()
+ {
+ // Multiple files, use zip64 even though it is not required
+ RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: true, forward_only: false);
+ }
+
+ [Fact]
+ public void Zip64_Single_Large_File_Fail()
+ {
+ try
+ {
+ // One single file, should fail
+ RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: false);
+ throw new Exception("Test did not fail?");
+ }
+ catch (NotSupportedException)
+ {
+ }
+ }
+
+ [Fact]
+ public void Zip64_Single_Large_File_Zip64_Streaming_Fail()
+ {
+ try
+ {
+ // One single file, should fail (fast) with zip64
+ RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: true);
+ throw new Exception("Test did not fail?");
+ }
+ catch (NotSupportedException)
+ {
+ }
+ }
+
+ [Fact]
+ public void Zip64_Single_Large_File_Streaming_Fail()
+ {
+ try
+ {
+ // One single file, should fail once the write discovers the problem
+ RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: true);
+ throw new Exception("Test did not fail?");
+ }
+ catch (NotSupportedException)
+ {
+ }
+ }
+
+ public void RunSingleTest(long files, long filesize, bool set_zip64, bool forward_only, long write_chunk_size = 1024 * 1024, string filename = "zip64-test.zip")
+ {
+ ResetScratch();
+ filename = Path.Combine(SCRATCH2_FILES_PATH, filename);
+
+ if (File.Exists(filename))
+ File.Delete(filename);
+
+ if (!File.Exists(filename))
+ CreateZipArchive(filename, files, filesize, write_chunk_size, set_zip64, forward_only);
+
+ var resForward = ReadForwardOnly(filename);
+ if (resForward.Item1 != files)
+ throw new Exception($"Incorrect number of items reported: {resForward.Item1}, should have been {files}");
+
+ if (resForward.Item2 != files * filesize)
+ throw new Exception($"Incorrect combined size reported: {resForward.Item2}, should have been {files * filesize}");
+
+ var resArchive = ReadArchive(filename);
+ if (resArchive.Item1 != files)
+ throw new Exception($"Incorrect number of items reported: {resArchive.Item1}, should have been {files}");
+ if (resArchive.Item2 != files * filesize)
+ throw new Exception($"Incorrect number of items reported: {resArchive.Item2}, should have been {files * filesize}");
+ }
+
+ public void CreateZipArchive(string filename, long files, long filesize, long chunksize, bool set_zip64, bool forward_only)
+ {
+ var data = new byte[chunksize];
+
+ // Use deflate for speed
+ var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = set_zip64 };
+
+ // Use no compression to ensure we hit the limits (actually inflates a bit, but seems better than using method==Store)
+ var eo = new ZipWriterEntryOptions() { DeflateCompressionLevel = SharpCompress.Compressors.Deflate.CompressionLevel.None };
+
+ using (var zip = File.OpenWrite(filename))
+ using(var st = forward_only ? (Stream)new NonSeekableStream(zip) : zip)
+ using (var zipWriter = (ZipWriter)WriterFactory.Open(st, ArchiveType.Zip, opts))
+ {
+
+ for (var i = 0; i < files; i++)
+ using (var str = zipWriter.WriteToStream(i.ToString(), eo))
+ {
+ var left = filesize;
+ while (left > 0)
+ {
+ var b = (int)Math.Min(left, data.Length);
+ str.Write(data, 0, b);
+ left -= b;
+ }
+ }
+ }
+ }
+
+ public Tuple ReadForwardOnly(string filename)
+ {
+ long count = 0;
+ long size = 0;
+ Common.Zip.ZipEntry prev = null;
+ using (var fs = File.OpenRead(filename))
+ using (var rd = ZipReader.Open(fs, new ReaderOptions() { LookForHeader = false }))
+ while (rd.MoveToNextEntry())
+ {
+ using (rd.OpenEntryStream())
+ { }
+
+ count++;
+ if (prev != null)
+ size += prev.Size;
+
+ prev = rd.Entry;
+ }
+
+ if (prev != null)
+ size += prev.Size;
+
+ return new Tuple(count, size);
+ }
+
+ public Tuple ReadArchive(string filename)
+ {
+ using (var archive = ArchiveFactory.Open(filename))
+ {
+ return new Tuple(
+ archive.Entries.Count(),
+ archive.Entries.Select(x => x.Size).Sum()
+ );
+ }
+ }
+
+ ///
+ /// Helper to create non-seekable streams from filestream
+ ///
+ private class NonSeekableStream : Stream
+ {
+ private readonly Stream stream;
+ public NonSeekableStream(Stream s) { stream = s; }
+ public override bool CanRead { get { return stream.CanRead; } }
+ public override bool CanSeek { get { return false; } }
+ public override bool CanWrite { get { return stream.CanWrite; } }
+ public override long Length { get { throw new NotImplementedException(); } }
+ public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } }
+ public override void Flush() { stream.Flush(); }
+
+ public override int Read(byte[] buffer, int offset, int count)
+ { return stream.Read(buffer, offset, count); }
+
+ public override long Seek(long offset, SeekOrigin origin)
+ { throw new NotImplementedException(); }
+
+ public override void SetLength(long value)
+ { throw new NotImplementedException(); }
+
+ public override void Write(byte[] buffer, int offset, int count)
+ { stream.Write(buffer, offset, count); }
+ }
+ }
+}
\ No newline at end of file