From 7ae1f64ee378ba4ff7a5acb1097f3d0fff7ee54a Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 26 Oct 2023 11:56:38 -0400 Subject: [PATCH] Move all hashing to Hasher --- CHANGELIST.md | 1 + MPF.Core/Hashing/Hasher.cs | 117 +++++++++++++++++- MPF.Core/InfoTool.cs | 112 ----------------- .../Modules/UmdImageCreator/Parameters.cs | 3 +- 4 files changed, 119 insertions(+), 114 deletions(-) diff --git a/CHANGELIST.md b/CHANGELIST.md index 84cc1a41..ebf178fe 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -4,6 +4,7 @@ - Add other non-cryptographic hashes - Avoid unncessary allocation in hashing - Make Hasher more consistent +- Move all hashing to Hasher ### 2.7.3 (2023-10-26) diff --git a/MPF.Core/Hashing/Hasher.cs b/MPF.Core/Hashing/Hasher.cs index c26c574b..5386b323 100644 --- a/MPF.Core/Hashing/Hasher.cs +++ b/MPF.Core/Hashing/Hasher.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Generic; +using System.IO; using System.IO.Hashing; using System.Linq; using System.Security.Cryptography; +using System.Threading.Tasks; using MPF.Core.Data; namespace MPF.Core.Hashing @@ -9,7 +12,7 @@ namespace MPF.Core.Hashing /// /// Async hashing class wraper /// - public class Hasher : IDisposable + public sealed class Hasher : IDisposable { #region Properties @@ -139,6 +142,118 @@ namespace MPF.Core.Hashing #region Hashing + /// + /// Get hashes from an input file path + /// + /// Path to the input file + /// True if hashing was successful, false otherwise +#if NET48 + public static bool GetFileHashes(string filename, out long size, out string crc32, out string md5, out string sha1) +#else + public static bool GetFileHashes(string filename, out long size, out string? crc32, out string? md5, out string? sha1) +#endif + { + // Set all initial values + size = -1; crc32 = null; md5 = null; sha1 = null; + + // If the file doesn't exist, we can't do anything + if (!File.Exists(filename)) + return false; + + // Set the file size + size = new FileInfo(filename).Length; + + // Open the input file + var input = File.OpenRead(filename); + + try + { + // Get a list of hashers to run over the buffer + var hashers = new List + { + new Hasher(Hash.CRC32), + new Hasher(Hash.MD5), + new Hasher(Hash.SHA1), + new Hasher(Hash.SHA256), + new Hasher(Hash.SHA384), + new Hasher(Hash.SHA512), + }; + + // Initialize the hashing helpers + var loadBuffer = new ThreadLoadBuffer(input); + int buffersize = 3 * 1024 * 1024; + byte[] buffer0 = new byte[buffersize]; + byte[] buffer1 = new byte[buffersize]; + + /* + Please note that some of the following code is adapted from + RomVault. This is a modified version of how RomVault does + threaded hashing. As such, some of the terminology and code + is the same, though variable names and comments may have + been tweaked to better fit this code base. + */ + + // Pre load the first buffer + long refsize = size; + int next = refsize > buffersize ? buffersize : (int)refsize; + input.Read(buffer0, 0, next); + int current = next; + refsize -= next; + bool bufferSelect = true; + + while (current > 0) + { + // Trigger the buffer load on the second buffer + next = refsize > buffersize ? buffersize : (int)refsize; + if (next > 0) + loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); + + byte[] buffer = bufferSelect ? buffer0 : buffer1; + + // Run hashes in parallel + Parallel.ForEach(hashers, h => h.Process(buffer, current)); + + // Wait for the load buffer worker, if needed + if (next > 0) + loadBuffer.Wait(); + + // Setup for the next hashing step + current = next; + refsize -= next; + bufferSelect = !bufferSelect; + } + + // Finalize all hashing helpers + loadBuffer.Finish(); + Parallel.ForEach(hashers, h => h.Terminate()); + + // Get the results + crc32 = hashers.First(h => h.HashType == Hash.CRC32).CurrentHashString; + //crc64 = hashers.First(h => h.HashType == Hash.CRC64).CurrentHashString; + md5 = hashers.First(h => h.HashType == Hash.MD5).CurrentHashString; + sha1 = hashers.First(h => h.HashType == Hash.SHA1).CurrentHashString; + //sha256 = hashers.First(h => h.HashType == Hash.SHA256).CurrentHashString; + //sha384 = hashers.First(h => h.HashType == Hash.SHA384).CurrentHashString; + //sha512 = hashers.First(h => h.HashType == Hash.SHA512).CurrentHashString; + //xxHash32 = hashers.First(h => h.HashType == Hash.XxHash32).CurrentHashString; + //xxHash64 = hashers.First(h => h.HashType == Hash.XxHash64).CurrentHashString; + + // Dispose of the hashers + loadBuffer.Dispose(); + hashers.ForEach(h => h.Dispose()); + + return true; + } + catch (IOException) + { + return false; + } + finally + { + input.Dispose(); + } + } + /// /// Process a buffer of some length with the internal hash algorithm /// diff --git a/MPF.Core/InfoTool.cs b/MPF.Core/InfoTool.cs index c11e1898..29db78f1 100644 --- a/MPF.Core/InfoTool.cs +++ b/MPF.Core/InfoTool.cs @@ -195,118 +195,6 @@ namespace MPF.Core } } - /// - /// Get hashes from an input file path - /// - /// Path to the input file - /// True if hashing was successful, false otherwise -#if NET48 - internal static bool GetFileHashes(string filename, out long size, out string crc32, out string md5, out string sha1) -#else - internal static bool GetFileHashes(string filename, out long size, out string? crc32, out string? md5, out string? sha1) -#endif - { - // Set all initial values - size = -1; crc32 = null; md5 = null; sha1 = null; - - // If the file doesn't exist, we can't do anything - if (!File.Exists(filename)) - return false; - - // Set the file size - size = new FileInfo(filename).Length; - - // Open the input file - var input = File.OpenRead(filename); - - try - { - // Get a list of hashers to run over the buffer - var hashers = new List - { - new Hasher(Hash.CRC32), - new Hasher(Hash.MD5), - new Hasher(Hash.SHA1), - new Hasher(Hash.SHA256), - new Hasher(Hash.SHA384), - new Hasher(Hash.SHA512), - }; - - // Initialize the hashing helpers - var loadBuffer = new ThreadLoadBuffer(input); - int buffersize = 3 * 1024 * 1024; - byte[] buffer0 = new byte[buffersize]; - byte[] buffer1 = new byte[buffersize]; - - /* - Please note that some of the following code is adapted from - RomVault. This is a modified version of how RomVault does - threaded hashing. As such, some of the terminology and code - is the same, though variable names and comments may have - been tweaked to better fit this code base. - */ - - // Pre load the first buffer - long refsize = size; - int next = refsize > buffersize ? buffersize : (int)refsize; - input.Read(buffer0, 0, next); - int current = next; - refsize -= next; - bool bufferSelect = true; - - while (current > 0) - { - // Trigger the buffer load on the second buffer - next = refsize > buffersize ? buffersize : (int)refsize; - if (next > 0) - loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); - - byte[] buffer = bufferSelect ? buffer0 : buffer1; - - // Run hashes in parallel - Parallel.ForEach(hashers, h => h.Process(buffer, current)); - - // Wait for the load buffer worker, if needed - if (next > 0) - loadBuffer.Wait(); - - // Setup for the next hashing step - current = next; - refsize -= next; - bufferSelect = !bufferSelect; - } - - // Finalize all hashing helpers - loadBuffer.Finish(); - Parallel.ForEach(hashers, h => h.Terminate()); - - // Get the results - crc32 = hashers.First(h => h.HashType == Hash.CRC32).CurrentHashString; - //crc64 = hashers.First(h => h.HashType == Hash.CRC64).CurrentHashString; - md5 = hashers.First(h => h.HashType == Hash.MD5).CurrentHashString; - sha1 = hashers.First(h => h.HashType == Hash.SHA1).CurrentHashString; - //sha256 = hashers.First(h => h.HashType == Hash.SHA256).CurrentHashString; - //sha384 = hashers.First(h => h.HashType == Hash.SHA384).CurrentHashString; - //sha512 = hashers.First(h => h.HashType == Hash.SHA512).CurrentHashString; - //xxHash32 = hashers.First(h => h.HashType == Hash.XxHash32).CurrentHashString; - //xxHash64 = hashers.First(h => h.HashType == Hash.XxHash64).CurrentHashString; - - // Dispose of the hashers - loadBuffer.Dispose(); - hashers.ForEach(h => h.Dispose()); - - return true; - } - catch (IOException) - { - return false; - } - finally - { - input.Dispose(); - } - } - /// /// Get the last modified date from a file path, if possible /// diff --git a/MPF.Core/Modules/UmdImageCreator/Parameters.cs b/MPF.Core/Modules/UmdImageCreator/Parameters.cs index b718d3a3..3193329b 100644 --- a/MPF.Core/Modules/UmdImageCreator/Parameters.cs +++ b/MPF.Core/Modules/UmdImageCreator/Parameters.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using MPF.Core.Converters; using MPF.Core.Data; +using MPF.Core.Hashing; using SabreTools.RedumpLib.Data; namespace MPF.Core.Modules.UmdImageCreator @@ -96,7 +97,7 @@ namespace MPF.Core.Modules.UmdImageCreator info.Extras!.PVD = GetPVD(basePath + "_mainInfo.txt") ?? string.Empty; #endif - if (InfoTool.GetFileHashes(basePath + ".iso", out long filesize, out var crc32, out var md5, out var sha1)) + if (Hasher.GetFileHashes(basePath + ".iso", out long filesize, out var crc32, out var md5, out var sha1)) { #if NET48 info.SizeAndChecksums.Size = filesize;