diff --git a/SabreTools.Hashing.Test/CompressedStreamTests.cs b/SabreTools.Hashing.Test/CompressedStreamTests.cs new file mode 100644 index 0000000..055e12d --- /dev/null +++ b/SabreTools.Hashing.Test/CompressedStreamTests.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using System.IO.Compression; +using Xunit; + +namespace SabreTools.Hashing.Test +{ + public class CompressedStreamTests + { + private static readonly string _hashFilePath = Path.Combine(Environment.CurrentDirectory, "TestData", "file-to-hash.zip"); + + #region Known File Information + + private const long _hashFileSize = 125; + private const string _adler32 = "08562d95"; +#if NET7_0_OR_GREATER + private const string _blake3 = "d4bd7ca6f1ebea9580d9381106b248eb5b6069170d0bfd00b17d659fcd10dcdc"; +#endif + private const string _crc16_ccitt = "482d"; + private const string _crc16_ibm = "7573"; + private const string _crc32 = "ba02a660"; + private const string _crc64 = "a0e0009c18b5338d"; + private const string _crc64_reversed = "fb49044e8331f6e5"; + private const string _fletcher16 = "46c1"; + private const string _fletcher32 = "073f2d94"; + private const string _md5 = "b722871eaa950016296184d026c5dec9"; +#if NETFRAMEWORK + private const string _ripemd160 = "346361e1d7fdb836650cecdb842b0dbe660eed66"; +#endif + private const string _sha1 = "eea1ee2d801d830c4bdad4df3c8da6f9f52d1a9f"; + private const string _sha256 = "fdb02dee8c319c52087382c45f099c90d0b6cc824850aff28c1bfb2884b7b855"; + private const string _sha384 = "e276c49618fff25bc1fe2e0659cd0ef0e7c1186563b063e07c52323b9899f3ce9b091be04d6208444b3ef1265e879074"; + private const string _sha512 = "15d69514eb628c2403e945a7cafd1d27e557f6e336c69b63ea17e7ed9d256cc374ee662f09305836d6de37fdae59d83883b982aa8446e4ff26346b6b6b50b240"; +#if NET8_0_OR_GREATER + private const string _sha3_256 = "1d76459e68c865b5911ada5104067cc604c5c60b345c4e81b3905e916a43c868"; + private const string _sha3_384 = "1bcbed87b73f25c0adf486c3afbf0ea3105763c387af3f8b2bd79b0a1964d42832b1d7c6a2225f9153ead26f442e8b67"; + private const string _sha3_512 = "89852144df37c58d01f5912124f1942dd00bac0346eb3971943416699c3094cff087fb42c356019c3d91f8e8f55b9254c8caec48e9414af6817297d06725ffeb"; + private const string _shake128 = "e5f88d0db79a71c39490beb9ebac21eaf4a5d6368438fca20f5e4ce77cfee9aa"; + private const string _shake256 = "24d9e83198bbc7baf4dcd293bfc35ae3fff05399786c37318f1b1ef85f41970c66926f8a2a1f912d96e2d8e45535af88a301a1c200697437c1a65d7e980344bc"; +#endif + private const string _spamsum = "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL"; +#if NET462_OR_GREATER || NETCOREAPP + private const string _xxhash32 = "8e331daa"; + private const string _xxhash64 = "082bf6f0a49e1e18"; + private const string _xxhash3 = "040474eb0eda9ff2"; + private const string _xxhash128 = "d934b4b4a5e1e11baeef8012fbcd11e8"; +#endif + + #endregion + + [Fact] + public void GetStreamHashesTest() + { + var zipFile = ZipFile.OpenRead(_hashFilePath); + var fileStream = zipFile.Entries[0].Open(); + var hashDict = HashTool.GetStreamHashes(fileStream); + + Assert.NotNull(hashDict); + Assert.Equal(_adler32, hashDict![HashType.Adler32]); +#if NET7_0_OR_GREATER + Assert.Equal(_blake3, hashDict[HashType.BLAKE3]); +#endif + Assert.Equal(_crc16_ccitt, hashDict[HashType.CRC16_CCITT]); + Assert.Equal(_crc16_ibm, hashDict[HashType.CRC16_IBM]); + Assert.Equal(_crc32, hashDict[HashType.CRC32]); + Assert.Equal(_crc32, hashDict[HashType.CRC32_ISO]); + Assert.Equal(_crc32, hashDict[HashType.CRC32_Naive]); + Assert.Equal(_crc32, hashDict[HashType.CRC32_Optimized]); + Assert.Equal(_crc32, hashDict[HashType.CRC32_Parallel]); + Assert.Equal(_crc64, hashDict[HashType.CRC64]); + Assert.Equal(_crc64_reversed, hashDict[HashType.CRC64_Reversed]); + Assert.Equal(_fletcher16, hashDict[HashType.Fletcher16]); + Assert.Equal(_fletcher32, hashDict[HashType.Fletcher32]); + Assert.Equal(_md5, hashDict[HashType.MD5]); + Assert.Equal(_sha1, hashDict[HashType.SHA1]); + Assert.Equal(_sha256, hashDict[HashType.SHA256]); + Assert.Equal(_sha384, hashDict[HashType.SHA384]); + Assert.Equal(_sha512, hashDict[HashType.SHA512]); +#if NET8_0_OR_GREATER + if (System.Security.Cryptography.SHA3_256.IsSupported) + Assert.Equal(_sha3_256, hashDict[HashType.SHA3_256]); + if (System.Security.Cryptography.SHA3_384.IsSupported) + Assert.Equal(_sha3_384, hashDict[HashType.SHA3_384]); + if (System.Security.Cryptography.SHA3_512.IsSupported) + Assert.Equal(_sha3_512, hashDict[HashType.SHA3_512]); + if (System.Security.Cryptography.Shake128.IsSupported) + Assert.Equal(_shake128, hashDict[HashType.SHAKE128]); + if (System.Security.Cryptography.Shake256.IsSupported) + Assert.Equal(_shake256, hashDict[HashType.SHAKE256]); +#endif + Assert.Equal(_spamsum, hashDict[HashType.SpamSum]); +#if NET462_OR_GREATER || NETCOREAPP + Assert.Equal(_xxhash32, hashDict[HashType.XxHash32]); + Assert.Equal(_xxhash64, hashDict[HashType.XxHash64]); + Assert.Equal(_xxhash3, hashDict[HashType.XxHash3]); + Assert.Equal(_xxhash128, hashDict[HashType.XxHash128]); +#endif + } + } +} \ No newline at end of file diff --git a/SabreTools.Hashing.Test/SabreTools.Hashing.Test.csproj b/SabreTools.Hashing.Test/SabreTools.Hashing.Test.csproj index 3161046..2fda06e 100644 --- a/SabreTools.Hashing.Test/SabreTools.Hashing.Test.csproj +++ b/SabreTools.Hashing.Test/SabreTools.Hashing.Test.csproj @@ -21,6 +21,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/SabreTools.Hashing.Test/TestData/file-to-hash.zip b/SabreTools.Hashing.Test/TestData/file-to-hash.zip new file mode 100644 index 0000000..89be615 Binary files /dev/null and b/SabreTools.Hashing.Test/TestData/file-to-hash.zip differ diff --git a/SabreTools.Hashing/HashTool.cs b/SabreTools.Hashing/HashTool.cs index 6780484..b33b5cc 100644 --- a/SabreTools.Hashing/HashTool.cs +++ b/SabreTools.Hashing/HashTool.cs @@ -338,7 +338,72 @@ namespace SabreTools.Hashing /// Stream to hash /// Array of hash types to get from the file /// Dictionary containing hashes on success, null on error - public static Dictionary? GetStreamHashes(Stream input, HashType[] hashTypes, long length = -1, bool leaveOpen = false) + public static Dictionary? GetStreamHashes(Stream input, HashType[] hashTypes, bool leaveOpen = false) + { + // Create the output dictionary + var hashDict = new Dictionary(); + + // Run the hashing + var hashers = GetStreamHashesInternal(input, hashTypes, leaveOpen); + if (hashers == null) + return null; + + // Get the results + foreach (var hasher in hashers) + { + hashDict[hasher.Key] = hasher.Value.CurrentHashString; + } + + // Dispose of the hashers + foreach (var hasher in hashers.Values) + { + hasher.Dispose(); + } + + return hashDict; + } + + /// + /// Get hashes from an input Stream + /// + /// Stream to hash + /// Array of hash types to get from the file + /// Dictionary containing hashes on success, null on error + public static Dictionary? GetStreamHashArrays(Stream input, HashType[] hashTypes, bool leaveOpen = false) + { + // Create the output dictionary + var hashDict = new Dictionary(); + + + + // Run the hashing + var hashers = GetStreamHashesInternal(input, hashTypes, leaveOpen); + if (hashers == null) + return null; + + // Get the results + foreach (var hasher in hashers) + { + hashDict[hasher.Key] = hasher.Value.CurrentHashBytes; + } + + // Dispose of the hashers + foreach (var hasher in hashers.Values) + { + hasher.Dispose(); + } + + return hashDict; + } + + /// + /// Get hashes from an input stream + /// + /// + /// + /// + /// + private static Dictionary? GetStreamHashesInternal(Stream input, HashType[] hashTypes, bool leaveOpen) { // Create the output dictionary var hashDict = new Dictionary(); @@ -369,40 +434,32 @@ namespace SabreTools.Hashing */ // Pre load the first buffer - long refsize = length > -1 ? length : input.Length; - int next = refsize > buffersize ? buffersize : (int)refsize; - input.Read(buffer0, 0, next); - int current = next; - refsize -= next; + int lastRead = input.Read(buffer0, 0, buffersize); bool bufferSelect = true; - while (current > 0) + while (lastRead > 0) { // Trigger the buffer load on the second buffer - next = refsize > buffersize ? buffersize : (int)refsize; - if (next > 0) - loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); - + loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, buffersize); byte[] buffer = bufferSelect ? buffer0 : buffer1; #if NET20 || NET35 // Run hashers sequentially on each chunk foreach (var h in hashers) { - h.Value.Process(buffer, 0, current); + h.Value.Process(buffer, 0, lastRead); } #else // Run hashers in parallel on each chunk - Parallel.ForEach(hashers, h => h.Value.Process(buffer, 0, current)); + Parallel.ForEach(hashers, h => h.Value.Process(buffer, 0, lastRead)); #endif // Wait for the load buffer worker, if needed - if (next > 0) + if (loadBuffer.SizeRead > 0) loadBuffer.Wait(); // Setup for the next hashing step - current = next; - refsize -= next; + lastRead = loadBuffer.SizeRead; bufferSelect = !bufferSelect; } @@ -417,131 +474,8 @@ namespace SabreTools.Hashing Parallel.ForEach(hashers, h => h.Value.Terminate()); #endif - // Get the results - foreach (var hasher in hashers) - { - hashDict[hasher.Key] = hasher.Value.CurrentHashString; - } - - // Dispose of the hashers loadBuffer.Dispose(); - foreach (var hasher in hashers.Values) - { - hasher.Dispose(); - } - - return hashDict; - } - catch (IOException) - { - return null; - } - finally - { - if (!leaveOpen) - input.Dispose(); - } - } - - /// - /// Get hashes from an input Stream - /// - /// Stream to hash - /// Array of hash types to get from the file - /// Dictionary containing hashes on success, null on error - private static Dictionary? GetStreamHashArrays(Stream input, HashType[] hashTypes, long length = -1, bool leaveOpen = false) - { - // Create the output dictionary - var hashDict = new Dictionary(); - - try - { - // Get a list of hashers to run over the buffer - var hashers = new Dictionary(); - - // Add hashers based on requested types - foreach (HashType hashType in hashTypes) - { - hashers[hashType] = new HashWrapper(hashType); - } - - // Initialize the hashing helpers - var loadBuffer = new ThreadLoadBuffer(input); - int buffersize = 3 * 1024 * 1024; - byte[] buffer0 = new byte[buffersize]; - byte[] buffer1 = new byte[buffersize]; - - /* - Please note that some of the following code is adapted from - RomVault. This is a modified version of how RomVault does - threaded hashing. As such, some of the terminology and code - is the same, though variable names and comments may have - been tweaked to better fit this code base. - */ - - // Pre load the first buffer - long refsize = length > -1 ? length : input.Length; - int next = refsize > buffersize ? buffersize : (int)refsize; - input.Read(buffer0, 0, next); - int current = next; - refsize -= next; - bool bufferSelect = true; - - while (current > 0) - { - // Trigger the buffer load on the second buffer - next = refsize > buffersize ? buffersize : (int)refsize; - if (next > 0) - loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); - - byte[] buffer = bufferSelect ? buffer0 : buffer1; - -#if NET20 || NET35 - // Run hashers sequentially on each chunk - foreach (var h in hashers) - { - h.Value.Process(buffer, 0, current); - } -#else - // Run hashers in parallel on each chunk - Parallel.ForEach(hashers, h => h.Value.Process(buffer, 0, current)); -#endif - - // Wait for the load buffer worker, if needed - if (next > 0) - loadBuffer.Wait(); - - // Setup for the next hashing step - current = next; - refsize -= next; - bufferSelect = !bufferSelect; - } - - // Finalize all hashing helpers - loadBuffer.Finish(); -#if NET20 || NET35 - foreach (var h in hashers) - { - h.Value.Terminate(); - } -#else - Parallel.ForEach(hashers, h => h.Value.Terminate()); -#endif - - // Get the results - foreach (var hasher in hashers) - { - hashDict[hasher.Key] = hasher.Value.CurrentHashBytes; - } - - // Dispose of the hashers - loadBuffer.Dispose(); - foreach (var hasher in hashers.Values) - { - hasher.Dispose(); - } - - return hashDict; + return hashers; } catch (IOException) { diff --git a/SabreTools.Hashing/Tiger/TigerHash.cs b/SabreTools.Hashing/Tiger/TigerHash.cs index 48135b0..41a8ea2 100644 --- a/SabreTools.Hashing/Tiger/TigerHash.cs +++ b/SabreTools.Hashing/Tiger/TigerHash.cs @@ -1,5 +1,4 @@ using System; -using System.ComponentModel.DataAnnotations; using System.Linq; namespace SabreTools.Hashing.Tiger