From 53393e744e3ed79df30e724774092f86f3b9df0f Mon Sep 17 00:00:00 2001 From: Brendan Grant Date: Sat, 13 Feb 2021 13:33:43 -0600 Subject: [PATCH 1/7] Supporting reading contents of incomplete files --- src/SharpCompress/Archives/Rar/RarArchive.cs | 5 +++-- src/SharpCompress/Archives/Rar/RarArchiveEntry.cs | 7 +++++-- src/SharpCompress/Archives/Rar/RarArchiveEntryFactory.cs | 6 ++++-- src/SharpCompress/Readers/ReaderOptions.cs | 2 ++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchive.cs b/src/SharpCompress/Archives/Rar/RarArchive.cs index b71b2147..0b191056 100644 --- a/src/SharpCompress/Archives/Rar/RarArchive.cs +++ b/src/SharpCompress/Archives/Rar/RarArchive.cs @@ -10,7 +10,8 @@ using SharpCompress.Readers.Rar; namespace SharpCompress.Archives.Rar { - public class RarArchive : AbstractArchive + public class + RarArchive : AbstractArchive { internal Lazy UnpackV2017 { get; } = new Lazy(() => new SharpCompress.Compressors.Rar.UnpackV2017.Unpack()); internal Lazy UnpackV1 { get; } = new Lazy(() => new SharpCompress.Compressors.Rar.UnpackV1.Unpack()); @@ -42,7 +43,7 @@ namespace SharpCompress.Archives.Rar protected override IEnumerable LoadEntries(IEnumerable volumes) { - return RarArchiveEntryFactory.GetEntries(this, volumes); + return RarArchiveEntryFactory.GetEntries(this, volumes, ReaderOptions); } protected override IEnumerable LoadVolumes(IEnumerable streams) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index eee9d996..c695a8b9 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -6,6 +6,7 @@ using SharpCompress.Common; using SharpCompress.Common.Rar; using SharpCompress.Common.Rar.Headers; using SharpCompress.Compressors.Rar; +using SharpCompress.Readers; namespace SharpCompress.Archives.Rar { @@ -13,11 +14,13 @@ namespace SharpCompress.Archives.Rar { private readonly ICollection parts; private readonly RarArchive archive; + private readonly ReaderOptions readerOptions; - internal RarArchiveEntry(RarArchive archive, IEnumerable parts) + internal RarArchiveEntry(RarArchive archive, IEnumerable parts, ReaderOptions readerOptions) { this.parts = parts.ToList(); this.archive = archive; + this.readerOptions = readerOptions; } public override CompressionType CompressionType => CompressionType.Rar; @@ -75,7 +78,7 @@ namespace SharpCompress.Archives.Rar private void CheckIncomplete() { - if (!IsComplete) + if (!readerOptions.DisableCheckIncomplete && !IsComplete) { throw new IncompleteArchiveException("ArchiveEntry is incomplete and cannot perform this operation."); } diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntryFactory.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntryFactory.cs index e41c024d..2d471c18 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntryFactory.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntryFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using SharpCompress.Common.Rar; +using SharpCompress.Readers; namespace SharpCompress.Archives.Rar { @@ -36,11 +37,12 @@ namespace SharpCompress.Archives.Rar } internal static IEnumerable GetEntries(RarArchive archive, - IEnumerable rarParts) + IEnumerable rarParts, + ReaderOptions readerOptions) { foreach (var groupedParts in GetMatchedFileParts(rarParts)) { - yield return new RarArchiveEntry(archive, groupedParts); + yield return new RarArchiveEntry(archive, groupedParts, readerOptions); } } } diff --git a/src/SharpCompress/Readers/ReaderOptions.cs b/src/SharpCompress/Readers/ReaderOptions.cs index 15302dc2..110b0c33 100644 --- a/src/SharpCompress/Readers/ReaderOptions.cs +++ b/src/SharpCompress/Readers/ReaderOptions.cs @@ -10,5 +10,7 @@ namespace SharpCompress.Readers public bool LookForHeader { get; set; } public string? Password { get; set; } + + public bool DisableCheckIncomplete { get; set; } } } \ No newline at end of file From 5b86c40d5b8ff493e701b9b8170b934b6451cc56 Mon Sep 17 00:00:00 2001 From: Brendan Grant Date: Sat, 13 Feb 2021 13:34:57 -0600 Subject: [PATCH 2/7] Properly detect if RAR is complete at the end or not --- src/SharpCompress/Archives/Rar/RarArchiveEntry.cs | 2 +- src/SharpCompress/Common/Rar/Headers/FileHeader.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index c695a8b9..dbdbd806 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -72,7 +72,7 @@ namespace SharpCompress.Archives.Rar { get { - return parts.Select(fp => fp.FileHeader).Any(fh => !fh.IsSplitAfter); + return parts.Select(fp => fp.FileHeader).Any(fh => !fh.IsSplitBefore && !fh.IsSplitAfter); } } diff --git a/src/SharpCompress/Common/Rar/Headers/FileHeader.cs b/src/SharpCompress/Common/Rar/Headers/FileHeader.cs index 11d7883f..2c361a09 100644 --- a/src/SharpCompress/Common/Rar/Headers/FileHeader.cs +++ b/src/SharpCompress/Common/Rar/Headers/FileHeader.cs @@ -437,6 +437,7 @@ namespace SharpCompress.Common.Rar.Headers internal long DataStartPosition { get; set; } public Stream PackedStream { get; set; } + public bool IsSplitBefore => IsRar5 ? HasHeaderFlag(HeaderFlagsV5.SPLIT_BEFORE) : HasFlag(FileFlagsV4.SPLIT_BEFORE); public bool IsSplitAfter => IsRar5 ? HasHeaderFlag(HeaderFlagsV5.SPLIT_AFTER) : HasFlag(FileFlagsV4.SPLIT_AFTER); public bool IsDirectory => HasFlag(IsRar5 ? FileFlagsV5.DIRECTORY : FileFlagsV4.DIRECTORY); From d1d2758ee07ad5a8f72d3a60a1f5f3e75076daab Mon Sep 17 00:00:00 2001 From: Lars Vahlenberg Date: Sat, 13 Feb 2021 23:57:03 +0100 Subject: [PATCH 3/7] Propsal for handling Zip with long comment --- .../Common/Zip/SeekableZipHeaderFactory.cs | 73 +++++++++++++----- .../SharpCompress.Test/Zip/ZipArchiveTests.cs | 12 +++ .../TestArchives/Archives/Zip.LongComment.zip | Bin 0 -> 4261 bytes 3 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 tests/TestArchives/Archives/Zip.LongComment.zip diff --git a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs index 59ed3060..86eb7d08 100644 --- a/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs +++ b/src/SharpCompress/Common/Zip/SeekableZipHeaderFactory.cs @@ -8,7 +8,10 @@ namespace SharpCompress.Common.Zip { internal sealed class SeekableZipHeaderFactory : ZipHeaderFactory { - private const int MAX_ITERATIONS_FOR_DIRECTORY_HEADER = 4096; + private const int MINIMUM_EOCD_LENGTH = 22; + private const int ZIP64_EOCD_LENGTH = 20; + // Comment may be within 64kb + structure 22 bytes + private const int MAX_SEARCH_LENGTH_FOR_EOCD = 65557; private bool _zip64; internal SeekableZipHeaderFactory(string? password, ArchiveEncoding archiveEncoding) @@ -20,14 +23,24 @@ namespace SharpCompress.Common.Zip { var reader = new BinaryReader(stream); - SeekBackToHeader(stream, reader, DIRECTORY_END_HEADER_BYTES); + SeekBackToHeader(stream, reader); + + var eocd_location = stream.Position; var entry = new DirectoryEndHeader(); entry.Read(reader); if (entry.IsZip64) { _zip64 = true; - SeekBackToHeader(stream, reader, ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR); + + // ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR should be before the EOCD + stream.Seek(eocd_location - ZIP64_EOCD_LENGTH - 4, SeekOrigin.Begin); + uint zip64_locator = reader.ReadUInt32(); + if( zip64_locator != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR ) + { + throw new ArchiveException("Failed to locate the Zip64 Directory Locator"); + } + var zip64Locator = new Zip64DirectoryEndLocatorHeader(); zip64Locator.Read(reader); @@ -73,27 +86,49 @@ namespace SharpCompress.Common.Zip } } - private static void SeekBackToHeader(Stream stream, BinaryReader reader, uint headerSignature) + private static bool IsMatch( byte[] haystack, int position, byte[] needle) { - long offset = 0; - uint signature; - int iterationCount = 0; - do + for( int i = 0; i < needle.Length; i++ ) { - if ((stream.Length + offset) - 4 < 0) + if( haystack[ position + i ] != needle[ i ] ) { - throw new ArchiveException("Failed to locate the Zip Header"); - } - stream.Seek(offset - 4, SeekOrigin.End); - signature = reader.ReadUInt32(); - offset--; - iterationCount++; - if (iterationCount > MAX_ITERATIONS_FOR_DIRECTORY_HEADER) - { - throw new ArchiveException("Could not find Zip file Directory at the end of the file. File may be corrupted."); + return false; } } - while (signature != headerSignature); + + return true; + } + private static void SeekBackToHeader(Stream stream, BinaryReader reader) + { + // Minimum EOCD length + if (stream.Length < MINIMUM_EOCD_LENGTH) + { + throw new ArchiveException("Could not find Zip file Directory at the end of the file. File may be corrupted."); + } + + int len = stream.Length < MAX_SEARCH_LENGTH_FOR_EOCD ? (int)stream.Length : MAX_SEARCH_LENGTH_FOR_EOCD; + // We search for marker in reverse to find the first occurance + byte[] needle = { 0x06, 0x05, 0x4b, 0x50 }; + + stream.Seek(-len, SeekOrigin.End); + + byte[] seek = reader.ReadBytes(len); + + // Search in reverse + Array.Reverse(seek); + + var max_search_area = len - MINIMUM_EOCD_LENGTH; + + for( int pos_from_end = 0; pos_from_end < max_search_area; ++pos_from_end) + { + if( IsMatch(seek, pos_from_end, needle) ) + { + stream.Seek(-pos_from_end, SeekOrigin.End); + return; + } + } + + throw new ArchiveException("Failed to locate the Zip Header"); } internal LocalEntryHeader GetLocalHeader(Stream stream, DirectoryEntryHeader directoryEntryHeader) diff --git a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs index 4ecfb07b..d1dd1d47 100644 --- a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs @@ -564,5 +564,17 @@ namespace SharpCompress.Test.Zip } } } + + [Fact] + public void Zip_LongComment_Read() + { + string zipPath = Path.Combine(TEST_ARCHIVES_PATH, "Zip.Longcomment.zip"); + + using(ZipArchive za = ZipArchive.Open(zipPath)) + { + var count = za.Entries.Count; + Assert.Equal(1, count); + } + } } } diff --git a/tests/TestArchives/Archives/Zip.LongComment.zip b/tests/TestArchives/Archives/Zip.LongComment.zip new file mode 100644 index 0000000000000000000000000000000000000000..3af586d6620b11b8c2e58109f9b30e5f4561425f GIT binary patch literal 4261 zcmWIWW@h1H00G?{y@4PahWQy77#tNsQY%Un+%j`g^-3yA0=yZS>=|$=SAptOfYNB1 zxfmc4j0_SC-r=sV%r0JIM5hD1S=m6k85tNE5*Zj6bQu^J7zKRti&AqHG7E}Ja}`qZ zbMlK6iZe?T5_3~abQF^F^NLfGOHxZpixd)5G7B<`lQZ+u6;g9DOY{_c^NW%)6;jhv zOB8Z4lTwTF6><}cixc$}+)9g+Qx!^cN{TX*Q;QYyGK(`I=IbaFmSz?!q?TqD=jNv< zs6Nq&)@f^TAKa%x^;u|jS>SRC0W zWgzz}6sM*rlqD8rmKN(N1eRtN7nY_%J(HSTQd+E#mzu1QUzD5)wl+~AH7zYOIkTj+ z2^4enpcvk zr{G$WnV74PnOBlplv0|jkd&E}ng?R!CW7>W++JD)G8Pm(;26ikeZW|npctv_Gnsa zQEqBpNolT*LJq{cWtk<3si4qGEiTDSRe%Jn4kYpti;_!`A}TL4S4SZ;MIkq_A~UzN z7!tQ&yCDIXm{*bt@&z;oz;O-^NoZW;Bqo;@gW@c`G*zJ(6hEMVP07pyc_FbxA+;#K zSWm$-Pa&}c64RiBqNfm?ngX?|1e`{66w-<^^U^bOauPur^HM=E2(nBE8m|f&sd*_y zsYRJ33VEeDIfm!u0?aXGr8xzqC5a`epm7~WOz0Kg{Yo_V`^S% zA~@I}*+3zu6qJ;86bdpDi&9I95@8t?5=w~R$;nJF1%;0bD0@O<37qmkQ2Ru zq)2!cMid5+1OrK^I`HBHRxE&$3nigiP8WI# zeu=rD7%WaLEKMv?NGZ)!NJ}kE&jcm#oXiqv5djl}BsxeufYj=sW(-haR9sq|oLP`r zf>b`kf&f-xL;VHzEI8W0B{Imn(43lw$l>4!(orbOEK4maN(2{osYQt;h~f#7RzVp( zu@sb3;fbiAD77RLRJw&_mL-A;0B9FIG_!$Sm64dNkdc^_S&|4Y z5_1wuQWK#;QCy;@;F1sWX-Q^Ya%M_tUWr0dPGVjPD6qg46DZj#l;!7?mK1;-3MuEH zQCU!0R9cK$V}X@tf+~oV%v4YY0+pk%LOU^80pi`z5(Rj^faY2q1$aT8S^+L2bQB6o zbILOF5{tm)9L%ARQW8{lD_=2U74w1xzw zW^k$oWm!nO7@Aqof(ViX;7t;cdqANJO$2%hp(P4wnR(!<3sg(N+M_y1DHoD#5=%fe zV{Soy5i~g=f)VTwP|!fjN9Ur%Vpx`eH06=fCQ|tdE*lg;1{dTcCWGPu+8{+34Nb_9 z>R%zX6e-Z*H4!*EKyuL92NW)P3P|Y|DUB*Xt1D=zqs9lcu7_rGXj2I4g4FcX;*vyY z`w$exur?yNegsu-C8_DDMNn@;+Xjh{UXcPc5}-z9=79@LaKjIlXFw*VWTxk378hsc zf*Zbi3J`z65(7*nG$W-}K+8W+%O?k3SAy$&qyz-+SQSAVzR2MK$qJx04LG+$(61=Gh@+G`O2G19eHZUl4!`oAcngh8i0L2Sf88mcY^@R?&mPKl?gM$=S zk)fq$P>BMqN1(w3>KY)5Sp*B@c1V?xnV4J(DpbLNqo)9CxxlJmurR1jhQ>HJZGZv} zQ71rC4I;CF`~nLsc)KP%v9Z;18OEsYW1Eh@(_MHx>c>?xVVy*%tKZBz^5dbja0mc9T literal 0 HcmV?d00001 From 566c49ce53dea080068df95f427b3df2d24d4880 Mon Sep 17 00:00:00 2001 From: Lars Vahlenberg Date: Sun, 14 Feb 2021 02:42:32 +0100 Subject: [PATCH 4/7] Proposal Zip64 requires version 4.5 Number of disks is 4 bytes and not 8 --- src/SharpCompress/Writers/Zip/ZipWriter.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index 8bf0faba..01047315 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -253,7 +253,7 @@ namespace SharpCompress.Writers.Zip BinaryPrimitives.WriteUInt64LittleEndian(intBuf, (ulong)recordlen); OutputStream.Write(intBuf); // Size of zip64 end of central directory record - BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 0); + BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 45); OutputStream.Write(intBuf.Slice(0, 2)); // Made by BinaryPrimitives.WriteUInt16LittleEndian(intBuf, 45); OutputStream.Write(intBuf.Slice(0, 2)); // Version needed @@ -278,8 +278,8 @@ namespace SharpCompress.Writers.Zip OutputStream.Write(intBuf.Slice(0, 4)); // Entry disk BinaryPrimitives.WriteUInt64LittleEndian(intBuf, (ulong)streamPosition + size); OutputStream.Write(intBuf); // Offset to the zip64 central directory - BinaryPrimitives.WriteUInt32LittleEndian(intBuf, 0); - OutputStream.Write(intBuf); // Number of disks + BinaryPrimitives.WriteUInt32LittleEndian(intBuf, 1); + OutputStream.Write(intBuf.Slice(0, 4)); // Number of disks streamPosition += recordlen + (4 + 4 + 8 + 4); streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streampositionvalue; From 045093f4537a3865fc434aee299cbefc3eff50a0 Mon Sep 17 00:00:00 2001 From: Lars Vahlenberg Date: Sun, 14 Feb 2021 10:26:26 +0100 Subject: [PATCH 5/7] Linux is case sensitive with files names --- tests/SharpCompress.Test/Zip/ZipArchiveTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs index d1dd1d47..d272aa9a 100644 --- a/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs +++ b/tests/SharpCompress.Test/Zip/ZipArchiveTests.cs @@ -568,7 +568,7 @@ namespace SharpCompress.Test.Zip [Fact] public void Zip_LongComment_Read() { - string zipPath = Path.Combine(TEST_ARCHIVES_PATH, "Zip.Longcomment.zip"); + string zipPath = Path.Combine(TEST_ARCHIVES_PATH, "Zip.LongComment.zip"); using(ZipArchive za = ZipArchive.Open(zipPath)) { From a51b56339a9c221a2bab2e21877c1295a5551c27 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 14 Feb 2021 13:00:43 +0000 Subject: [PATCH 6/7] Fix complete entry check for RAR files. --- src/SharpCompress/Archives/Rar/RarArchiveEntry.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index dbdbd806..9f9dee0c 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -72,7 +72,8 @@ namespace SharpCompress.Archives.Rar { get { - return parts.Select(fp => fp.FileHeader).Any(fh => !fh.IsSplitBefore && !fh.IsSplitAfter); + var headers = parts.Select(x => x.FileHeader); + return !headers.First().IsSplitBefore && !headers.Last().IsSplitAfter; } } From 403baf05a671eb30580453aa6e6a22875096dddc Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 14 Feb 2021 13:07:35 +0000 Subject: [PATCH 7/7] Mark for 0.28 --- src/SharpCompress/SharpCompress.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SharpCompress/SharpCompress.csproj b/src/SharpCompress/SharpCompress.csproj index 782f3651..7dbd195f 100644 --- a/src/SharpCompress/SharpCompress.csproj +++ b/src/SharpCompress/SharpCompress.csproj @@ -2,9 +2,9 @@ SharpCompress - Pure C# Decompression/Compression en-US - 0.27.1 - 0.27.1 - 0.27.1 + 0.28.0 + 0.28.0 + 0.28.0 Adam Hathcock netstandard2.0;netstandard2.1;netcoreapp3.1;net5.0 true