From 3df7842d237bfd8d1bac85847112cdbba6273ba2 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 15 Jan 2021 12:13:27 -0800 Subject: [PATCH] Add ability to hash a file to base parameters --- MPF.Library/Data/BaseParameters.cs | 110 ++++++++++++++++++ MPF.Library/{Utilities => Hashing}/Hasher.cs | 39 ++++++- .../{Utilities => Hashing}/OptimizedCRC.cs | 0 MPF.Library/Hashing/ThreadLoadBuffer.cs | 79 +++++++++++++ 4 files changed, 226 insertions(+), 2 deletions(-) rename MPF.Library/{Utilities => Hashing}/Hasher.cs (74%) rename MPF.Library/{Utilities => Hashing}/OptimizedCRC.cs (100%) create mode 100644 MPF.Library/Hashing/ThreadLoadBuffer.cs diff --git a/MPF.Library/Data/BaseParameters.cs b/MPF.Library/Data/BaseParameters.cs index 1a4a13e3..e9b1bc9b 100644 --- a/MPF.Library/Data/BaseParameters.cs +++ b/MPF.Library/Data/BaseParameters.cs @@ -2,8 +2,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Compress.ThreadReaders; +using MPF.Hashing; using MPF.Utilities; using MPF.Web; @@ -384,6 +387,113 @@ namespace MPF.Data #region Common Information Extraction + /// + /// Get hashes from an input file path + /// + /// Path to the input file + /// True if hashing was successful, false otherwise + protected static bool GetFileHashes(string filename, out long size, out string crc32, out string md5, out string sha1) + { + // Set all initial values + size = -1; crc32 = null; md5 = null; sha1 = null; + + // If the file doesn't exist, we can't do anything + if (!File.Exists(filename)) + return false; + + // Set the file size + size = new FileInfo(filename).Length; + + // Open the input file + var input = File.OpenRead(filename); + + try + { + // Get a list of hashers to run over the buffer + List hashers = new List + { + new Hasher(Hash.CRC), + new Hasher(Hash.MD5), + new Hasher(Hash.SHA1), + new Hasher(Hash.SHA256), + new Hasher(Hash.SHA384), + new Hasher(Hash.SHA512), + }; + + // Initialize the hashing helpers + var loadBuffer = new ThreadLoadBuffer(input); + int buffersize = 3 * 1024 * 1024; + byte[] buffer0 = new byte[buffersize]; + byte[] buffer1 = new byte[buffersize]; + + /* + Please note that some of the following code is adapted from + RomVault. This is a modified version of how RomVault does + threaded hashing. As such, some of the terminology and code + is the same, though variable names and comments may have + been tweaked to better fit this code base. + */ + + // Pre load the first buffer + long refsize = size; + int next = refsize > buffersize ? buffersize : (int)refsize; + input.Read(buffer0, 0, next); + int current = next; + refsize -= next; + bool bufferSelect = true; + + while (current > 0) + { + // Trigger the buffer load on the second buffer + next = refsize > buffersize ? buffersize : (int)refsize; + if (next > 0) + loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); + + byte[] buffer = bufferSelect ? buffer0 : buffer1; + + // Run hashes in parallel + Parallel.ForEach(hashers, h => h.Process(buffer, current)); + + // Wait for the load buffer worker, if needed + if (next > 0) + loadBuffer.Wait(); + + // Setup for the next hashing step + current = next; + refsize -= next; + bufferSelect = !bufferSelect; + } + + // Finalize all hashing helpers + loadBuffer.Finish(); + Parallel.ForEach(hashers, h => h.Terminate()); + + // Get the results + crc32 = hashers.First(h => h.HashType == Hash.CRC).GetHashString(); + md5 = hashers.First(h => h.HashType == Hash.MD5).GetHashString(); + sha1 = hashers.First(h => h.HashType == Hash.SHA1).GetHashString(); + //sha256 = hashers.First(h => h.HashType == Hash.SHA256).GetHashString(); + //sha384 = hashers.First(h => h.HashType == Hash.SHA384).GetHashString(); + //sha512 = hashers.First(h => h.HashType == Hash.SHA512).GetHashString(); + + // Dispose of the hashers + loadBuffer.Dispose(); + hashers.ForEach(h => h.Dispose()); + + return true; + } + catch (IOException ex) + { + return false; + } + finally + { + input.Dispose(); + } + + return false; + } + /// /// Get the split values for ISO-based media /// diff --git a/MPF.Library/Utilities/Hasher.cs b/MPF.Library/Hashing/Hasher.cs similarity index 74% rename from MPF.Library/Utilities/Hasher.cs rename to MPF.Library/Hashing/Hasher.cs index 133674c8..47ed40a8 100644 --- a/MPF.Library/Utilities/Hasher.cs +++ b/MPF.Library/Hashing/Hasher.cs @@ -4,7 +4,7 @@ using System.Security.Cryptography; using MPF.Data; using OptimizedCRC; -namespace MPF.Utilities +namespace MPF.Hashing { /// /// Async hashing class wraper @@ -82,7 +82,7 @@ namespace MPF.Utilities /// /// Finalize the internal hash algorigthm /// - public void Finalize() + public void Terminate() { byte[] emptyBuffer = new byte[0]; switch (HashType) @@ -121,5 +121,40 @@ namespace MPF.Utilities return null; } + + /// + /// Get internal hash as a string + /// + public string GetHashString() + { + byte[] hash = GetHash(); + if (hash == null) + return null; + + return ByteArrayToString(hash); + } + + /// + /// Convert a byte array to a hex string + /// + /// Byte array to convert + /// Hex string representing the byte array + /// http://stackoverflow.com/questions/311165/how-do-you-convert-byte-array-to-hexadecimal-string-and-vice-versa + private static string ByteArrayToString(byte[] bytes) + { + // If we get null in, we send null out + if (bytes == null) + return null; + + try + { + string hex = BitConverter.ToString(bytes); + return hex.Replace("-", string.Empty).ToLowerInvariant(); + } + catch + { + return null; + } + } } } diff --git a/MPF.Library/Utilities/OptimizedCRC.cs b/MPF.Library/Hashing/OptimizedCRC.cs similarity index 100% rename from MPF.Library/Utilities/OptimizedCRC.cs rename to MPF.Library/Hashing/OptimizedCRC.cs diff --git a/MPF.Library/Hashing/ThreadLoadBuffer.cs b/MPF.Library/Hashing/ThreadLoadBuffer.cs new file mode 100644 index 00000000..10633554 --- /dev/null +++ b/MPF.Library/Hashing/ThreadLoadBuffer.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using System.Threading; + +namespace Compress.ThreadReaders +{ + public class ThreadLoadBuffer : IDisposable + { + private readonly AutoResetEvent _waitEvent; + private readonly AutoResetEvent _outEvent; + private readonly Thread _tWorker; + + private byte[] _buffer; + private int _size; + private readonly Stream _ds; + private bool _finished; + public bool errorState; + + public int SizeRead; + + public ThreadLoadBuffer(Stream ds) + { + _waitEvent = new AutoResetEvent(false); + _outEvent = new AutoResetEvent(false); + _finished = false; + _ds = ds; + errorState = false; + + _tWorker = new Thread(MainLoop); + _tWorker.Start(); + } + + public void Dispose() + { + _waitEvent.Close(); + _outEvent.Close(); + } + + private void MainLoop() + { + while (true) + { + _waitEvent.WaitOne(); + if (_finished) + { + break; + } + try + { + SizeRead = _ds.Read(_buffer, 0, _size); + } + catch (Exception) + { + errorState = true; + } + _outEvent.Set(); + } + } + + public void Trigger(byte[] buffer, int size) + { + _buffer = buffer; + _size = size; + _waitEvent.Set(); + } + + public void Wait() + { + _outEvent.WaitOne(); + } + + public void Finish() + { + _finished = true; + _waitEvent.Set(); + _tWorker.Join(); + } + } +} \ No newline at end of file