From 388bbe047e7be4a89eb3e6171c1cc3a44ce348e1 Mon Sep 17 00:00:00 2001 From: Lars Vahlenberg Date: Tue, 2 Jan 2024 20:46:55 +0100 Subject: [PATCH] Blake2 Archive test OK --- .../Archives/Rar/RarArchiveEntry.cs | 4 +- .../Common/Rar/Headers/FileHeader.cs | 28 +- src/SharpCompress/Common/Rar/RarEntry.cs | 2 +- .../Rar/MultiVolumeReadOnlyStream.cs | 2 +- .../Compressors/Rar/RarBLAKE2spStream.cs | 288 ++++++++++++++++++ .../Compressors/Rar/RarCrcStream.cs | 4 +- src/SharpCompress/Readers/Rar/RarReader.cs | 6 + .../SharpCompress.Test/Rar/RarArchiveTests.cs | 3 + .../SharpCompress.Test/Rar/RarReaderTests.cs | 21 +- 9 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 5885210a..1ad252d1 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -1,3 +1,5 @@ + +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -40,7 +42,7 @@ public class RarArchiveEntry : RarEntry, IArchiveEntry get { CheckIncomplete(); - return parts.Select(fp => fp.FileHeader).Single(fh => !fh.IsSplitAfter).FileCrc; + return BitConverter.ToUInt32( parts.Select(fp => fp.FileHeader).Single(fh => !fh.IsSplitAfter).FileCrc, 0); } } diff --git a/src/SharpCompress/Common/Rar/Headers/FileHeader.cs b/src/SharpCompress/Common/Rar/Headers/FileHeader.cs index 3775f036..57cd5c71 100644 --- a/src/SharpCompress/Common/Rar/Headers/FileHeader.cs +++ b/src/SharpCompress/Common/Rar/Headers/FileHeader.cs @@ -18,7 +18,7 @@ namespace SharpCompress.Common.Rar.Headers; internal class FileHeader : RarHeader { - private uint _fileCrc; + private byte[] _hash; public FileHeader(RarHeader header, RarCrcBinaryReader reader, HeaderType headerType) : base(header, reader, headerType) { } @@ -53,7 +53,7 @@ internal class FileHeader : RarHeader if (HasFlag(FileFlagsV5.HAS_CRC32)) { - FileCrc = reader.ReadUInt32(); + FileCrc = reader.ReadBytes(4); } var compressionInfo = reader.ReadRarVIntUInt16(); @@ -136,14 +136,12 @@ internal class FileHeader : RarHeader { const uint FHEXTRA_HASH_BLAKE2 = 0x0; - const uint HASH_BLAKE2 = 0x03; + // const uint HASH_BLAKE2 = 0x03; const int BLAKE2_DIGEST_SIZE = 0x20; if ((uint)reader.ReadRarVInt() == FHEXTRA_HASH_BLAKE2) { - var hash = HASH_BLAKE2; - var digest = reader.ReadBytes(BLAKE2_DIGEST_SIZE); - - throw new InvalidFormatException("Not yet implemented " + hash); + // var hash = HASH_BLAKE2; + _hash = reader.ReadBytes(BLAKE2_DIGEST_SIZE); } // enum HASH_TYPE {HASH_NONE,HASH_RAR14,HASH_CRC32,HASH_BLAKE2}; } @@ -242,7 +240,7 @@ internal class FileHeader : RarHeader HostOs = reader.ReadByte(); - FileCrc = reader.ReadUInt32(); + FileCrc = reader.ReadBytes(4); FileLastModifiedTime = Utility.DosDateToDateTime(reader.ReadUInt32()); @@ -414,18 +412,10 @@ internal class FileHeader : RarHeader private bool HasFlag(ushort flag) => (Flags & flag) == flag; - internal uint FileCrc + internal byte[] FileCrc { - get - { - if (IsRar5 && !HasFlag(FileFlagsV5.HAS_CRC32)) - { - //!!! rar5: - throw new InvalidOperationException("TODO rar5"); - } - return _fileCrc; - } - private set => _fileCrc = value; + get => _hash; + private set => _hash = value; } // 0 - storing diff --git a/src/SharpCompress/Common/Rar/RarEntry.cs b/src/SharpCompress/Common/Rar/RarEntry.cs index 7f280e4a..92149e14 100644 --- a/src/SharpCompress/Common/Rar/RarEntry.cs +++ b/src/SharpCompress/Common/Rar/RarEntry.cs @@ -20,7 +20,7 @@ public abstract class RarEntry : Entry /// /// The File's 32 bit CRC Hash /// - public override long Crc => FileHeader.FileCrc; + public override long Crc => BitConverter.ToUInt32( FileHeader.FileCrc, 0 ); /// /// The path of the file internal to the Rar Archive. diff --git a/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs b/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs index 1e4adfde..460d92eb 100644 --- a/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs +++ b/src/SharpCompress/Compressors/Rar/MultiVolumeReadOnlyStream.cs @@ -128,7 +128,7 @@ internal sealed class MultiVolumeReadOnlyStream : Stream public override bool CanWrite => false; - public uint CurrentCrc { get; private set; } + public byte[] CurrentCrc { get; private set; } public override void Flush() => throw new NotSupportedException(); diff --git a/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs new file mode 100644 index 00000000..5c6c5ce2 --- /dev/null +++ b/src/SharpCompress/Compressors/Rar/RarBLAKE2spStream.cs @@ -0,0 +1,288 @@ + +using System; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using SharpCompress.Common; +using SharpCompress.Common.Rar.Headers; + +namespace SharpCompress.Compressors.Rar; + +internal class RarBLAKE2spStream : RarStream +{ + private readonly MultiVolumeReadOnlyStream readStream; + + const uint BLAKE2S_NUM_ROUNDS = 10; + const uint BLAKE2S_FINAL_FLAG = (~(uint)0); + const int BLAKE2S_BLOCK_SIZE = 64; + const int BLAKE2S_DIGEST_SIZE = 32; + const int BLAKE2SP_PARALLEL_DEGREE = 8; + const uint BLAKE2S_INIT_IV_SIZE = 8; + + static readonly UInt32[] k_BLAKE2S_IV = { + 0x6A09E667U, 0xBB67AE85U, 0x3C6EF372U, 0xA54FF53AU, + 0x510E527FU, 0x9B05688CU, 0x1F83D9ABU, 0x5BE0CD19U + }; + + static readonly byte[][] k_BLAKE2S_Sigma = { + new byte[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + new byte[]{ 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + new byte[]{ 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + new byte[]{ 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + new byte[]{ 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + new byte[]{ 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + new byte[]{ 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + new byte[]{ 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + new byte[]{ 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + new byte[]{ 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 }, + }; + + internal class BLAKE2S + { + internal UInt32[] h; + internal UInt32[] t; + internal UInt32[] f; + internal byte[] b; + internal int bufferPosition; + internal UInt32 lastNodeFlag; + UInt32[] dummy; + + public BLAKE2S() + { + h = new uint[BLAKE2S_INIT_IV_SIZE]; + t = new uint[2]; + f = new uint[2]; + b = new byte[BLAKE2S_BLOCK_SIZE]; + dummy = new uint[2]; + } + }; + + internal class BLAKE2SP + { + internal BLAKE2S[] S; + internal int bufferPosition; + + public BLAKE2SP() + { + S = new BLAKE2S[BLAKE2SP_PARALLEL_DEGREE]; + for (int i = 0; i < S.Length; i++) + { + S[i] = new BLAKE2S(); + } + } + }; + + BLAKE2SP _blake2sp; + + byte[] _hash = { }; + + public RarBLAKE2spStream( + IRarUnpack unpack, + FileHeader fileHeader, + MultiVolumeReadOnlyStream readStream + ) + : base(unpack, fileHeader, readStream) + { + this.readStream = readStream; + this._hash = fileHeader.FileCrc; + this._blake2sp = new BLAKE2SP(); + ResetCrc(); + } + + public byte[] GetCrc() => _hash; + + internal void ResetCrc(BLAKE2S hash) + { + for (UInt32 j = 0; j < BLAKE2S_INIT_IV_SIZE; j++) + { + hash.h[j] = k_BLAKE2S_IV[j]; + } + hash.t[0] = 0; + hash.t[1] = 0; + hash.f[0] = 0; + hash.f[1] = 0; + hash.bufferPosition = 0; + hash.lastNodeFlag = 0; + } + + internal void G(ref UInt32[] m, ref byte[] sigma, int i, ref UInt32 a, ref UInt32 b, ref UInt32 c, ref UInt32 d) + { + a += b + m[sigma[2 * i]]; + d ^= a; + d = ( d >> 16) | ( d << 16 ); + c += d; + b ^= c; + b = ( b >> 12 ) | ( b << 20 ); + + a += b + m[sigma[2 * i + 1]]; + d ^= a; + d = ( d >> 8 ) | ( d << 24 ); + c += d; + b ^= c; + b = (b >> 7) | (b << 25); + } + + internal void Compress(BLAKE2S hash) + { + UInt32[] m = new UInt32[16]; + UInt32[] v = new UInt32[16]; + + for (int i = 0; i < 16; i++) + { + m[i] = BitConverter.ToUInt32(hash.b, i * 4); + } + + for (int i = 0; i < 8; i++) + { + v[i] = hash.h[i]; + } + + v[8] = k_BLAKE2S_IV[0]; + v[9] = k_BLAKE2S_IV[1]; + v[10] = k_BLAKE2S_IV[2]; + v[11] = k_BLAKE2S_IV[3]; + + v[12] = hash.t[0] ^ k_BLAKE2S_IV[4]; + v[13] = hash.t[1] ^ k_BLAKE2S_IV[5]; + v[14] = hash.f[0] ^ k_BLAKE2S_IV[6]; + v[15] = hash.f[1] ^ k_BLAKE2S_IV[7]; + + for( int r = 0; r < BLAKE2S_NUM_ROUNDS; r++) + { + ref byte[] sigma = ref k_BLAKE2S_Sigma[r]; + + G(ref m, ref sigma, 0, ref v[0], ref v[4], ref v[8], ref v[12]); + G(ref m, ref sigma, 1, ref v[1], ref v[5], ref v[9], ref v[13]); + G(ref m, ref sigma, 2, ref v[2], ref v[6], ref v[10], ref v[14]); + G(ref m, ref sigma, 3, ref v[3], ref v[7], ref v[11], ref v[15]); + G(ref m, ref sigma, 4, ref v[0], ref v[5], ref v[10], ref v[15]); + G(ref m, ref sigma, 5, ref v[1], ref v[6], ref v[11], ref v[12]); + G(ref m, ref sigma, 6, ref v[2], ref v[7], ref v[8], ref v[13]); + G(ref m, ref sigma, 7, ref v[3], ref v[4], ref v[9], ref v[14]); + } + + for( int i = 0; i < 8; i++ ) + { + hash.h[i] ^= v[i] ^ v[i + 8]; + } + } + + internal void Update(BLAKE2S hash, ReadOnlySpan data, int size) + { + int i = 0; + while(size != 0) + { + var pos = hash.bufferPosition; + var reminder = BLAKE2S_BLOCK_SIZE - pos; + + if(size <= reminder ) + { + data.Slice(i, size).CopyTo(new Span(hash.b, pos, size)); + hash.bufferPosition += size; + return; + } + data.Slice(i, reminder).CopyTo(new Span(hash.b, pos, reminder)); + hash.t[0] += BLAKE2S_BLOCK_SIZE; + hash.t[1] += hash.t[0] < BLAKE2S_BLOCK_SIZE ? 1U : 0U; + Compress(hash); + hash.bufferPosition = 0; + i += reminder; + size -= reminder; + } + } + + internal byte[] Final(BLAKE2S hash) + { + hash.t[0] += (uint)hash.bufferPosition; + hash.t[1] += hash.t[0] < hash.bufferPosition ? 1U : 0U; + hash.f[0] = BLAKE2S_FINAL_FLAG; + hash.f[1] = hash.lastNodeFlag; + Array.Clear(hash.b, hash.bufferPosition, BLAKE2S_BLOCK_SIZE - hash.bufferPosition); + Compress(hash); + + var mem = new MemoryStream(); + + for(int i = 0; i < 8; i++) + { + mem.Write( BitConverter.GetBytes(hash.h[i]), 0, 4 ); + } + + return mem.ToArray(); + } + + public void ResetCrc() + { + _blake2sp.bufferPosition = 0; + + for( UInt32 i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++ ) + { + _blake2sp.S[i].bufferPosition = 0; + ResetCrc(_blake2sp.S[i]); + _blake2sp.S[i].h[0] ^= (BLAKE2S_DIGEST_SIZE | BLAKE2SP_PARALLEL_DEGREE << 16 | 2 << 24); + _blake2sp.S[i].h[2] ^= i; + _blake2sp.S[i].h[3] ^= (BLAKE2S_DIGEST_SIZE << 24); + } + + _blake2sp.S[BLAKE2SP_PARALLEL_DEGREE - 1].lastNodeFlag = BLAKE2S_FINAL_FLAG; + } + + internal void Update(BLAKE2SP hash, ReadOnlySpan data, int size) + { + int i = 0; + var pos = hash.bufferPosition; + while( size != 0 ) + { + var index = pos / BLAKE2S_BLOCK_SIZE; + var reminder = BLAKE2S_BLOCK_SIZE - (pos & ( BLAKE2S_BLOCK_SIZE -1 ) ); + if( reminder > size ) + { + reminder = size; + } + // Update(hash.S[index], data, size); + Update(hash.S[index], data.Slice(i,reminder), reminder); + size -= reminder; + i += reminder; + pos += reminder; + pos &= (BLAKE2S_BLOCK_SIZE * (BLAKE2SP_PARALLEL_DEGREE - 1)); + } + hash.bufferPosition = pos; + } + + internal byte[] Final(BLAKE2SP hash) + { + var h = new BLAKE2S(); + + ResetCrc(h); + h.h[0] ^= (BLAKE2S_DIGEST_SIZE | BLAKE2SP_PARALLEL_DEGREE << 16 | 2 << 24); + h.h[3] ^= (1 << 16 | BLAKE2S_DIGEST_SIZE << 24); + h.lastNodeFlag = BLAKE2S_FINAL_FLAG; + + for(int i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) + { + var digest = Final(_blake2sp.S[i]); + Update(h, digest, BLAKE2S_DIGEST_SIZE); + } + + return Final(h); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var result = base.Read(buffer, offset, count); + if (result != 0) + { + Update(_blake2sp, new ReadOnlySpan(buffer,offset,result), result); + } + else + { + _hash = Final(_blake2sp); + if (!(GetCrc().SequenceEqual(readStream.CurrentCrc)) && count != 0) + { + // NOTE: we use the last FileHeader in a multipart volume to check CRC + throw new InvalidFormatException("file crc mismatch"); + } + } + + return result; + } +} diff --git a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs index 98d50b00..372abb43 100644 --- a/src/SharpCompress/Compressors/Rar/RarCrcStream.cs +++ b/src/SharpCompress/Compressors/Rar/RarCrcStream.cs @@ -1,3 +1,5 @@ + +using System; using SharpCompress.Common; using SharpCompress.Common.Rar.Headers; @@ -30,7 +32,7 @@ internal class RarCrcStream : RarStream { currentCrc = RarCRC.CheckCrc(currentCrc, buffer, offset, result); } - else if (GetCrc() != readStream.CurrentCrc && count != 0) + else if (GetCrc() != BitConverter.ToUInt32(readStream.CurrentCrc, 0) && count != 0) { // NOTE: we use the last FileHeader in a multipart volume to check CRC throw new InvalidFormatException("file crc mismatch"); diff --git a/src/SharpCompress/Readers/Rar/RarReader.cs b/src/SharpCompress/Readers/Rar/RarReader.cs index da8c132b..695d47b0 100644 --- a/src/SharpCompress/Readers/Rar/RarReader.cs +++ b/src/SharpCompress/Readers/Rar/RarReader.cs @@ -72,6 +72,12 @@ public abstract class RarReader : AbstractReader { return CreateEntryStream(new RarCrcStream(UnpackV1.Value, Entry.FileHeader, stream)); } + + if (Entry.FileHeader.FileCrc.Length > 5) + { + return CreateEntryStream(new RarBLAKE2spStream(UnpackV2017.Value, Entry.FileHeader, stream)); + } + return CreateEntryStream(new RarCrcStream(UnpackV2017.Value, Entry.FileHeader, stream)); } } diff --git a/tests/SharpCompress.Test/Rar/RarArchiveTests.cs b/tests/SharpCompress.Test/Rar/RarArchiveTests.cs index acc5bdc8..42e76299 100644 --- a/tests/SharpCompress.Test/Rar/RarArchiveTests.cs +++ b/tests/SharpCompress.Test/Rar/RarArchiveTests.cs @@ -608,4 +608,7 @@ public class RarArchiveTests : ArchiveTests Assert.True(archive.IsMultipartVolume()); Assert.False(archive.IsFirstVolume()); } + + [Fact] + public void Rar5_CRC_Blake2_Archive() => ArchiveFileRead("Rar5.crc_blake2.rar"); } diff --git a/tests/SharpCompress.Test/Rar/RarReaderTests.cs b/tests/SharpCompress.Test/Rar/RarReaderTests.cs index 1fdbeef9..fc78b01c 100644 --- a/tests/SharpCompress.Test/Rar/RarReaderTests.cs +++ b/tests/SharpCompress.Test/Rar/RarReaderTests.cs @@ -171,33 +171,30 @@ public class RarReaderTests : ReaderTests [Fact] public void Rar5_Reader() => Read("Rar5.rar", CompressionType.Rar); + [Fact] + public void Rar5_CRC_Blake2_Reader() => Read("Rar5.crc_blake2.rar", CompressionType.Rar); + [Fact] public void Rar_EncryptedFileAndHeader_Reader() => ReadRar("Rar.encrypted_filesAndHeader.rar", "test"); - /*[Fact] - public void Rar5_EncryptedFileAndHeader_Reader() - { + [Fact] + public void Rar5_EncryptedFileAndHeader_Reader() => ReadRar("Rar5.encrypted_filesAndHeader.rar", "test"); - }*/ [Fact] public void Rar_EncryptedFileOnly_Reader() => ReadRar("Rar.encrypted_filesOnly.rar", "test"); - /*[Fact] - public void Rar5_EncryptedFileOnly_Reader() - { + [Fact] + public void Rar5_EncryptedFileOnly_Reader() => ReadRar("Rar5.encrypted_filesOnly.rar", "test"); - }*/ [Fact] public void Rar_Encrypted_Reader() => ReadRar("Rar.Encrypted.rar", "test"); - /*[Fact] - public void Rar5_Encrypted_Reader() - { + [Fact] + public void Rar5_Encrypted_Reader() => ReadRar("Rar5.encrypted_filesOnly.rar", "test"); - }*/ private void ReadRar(string testArchive, string password) => Read(testArchive, CompressionType.Rar, new ReaderOptions { Password = password });