SabreTools/MPF
1
0
Fork 0
mirror of https://github.com/SabreTools/MPF.git synced 2026-07-02 17:24:48 +00:00
Code Issues 392 Packages Projects Releases 20 Wiki Activity

Use SabreTools.Hashing

Browse Source
This commit is contained in:
Matt Nadareski
2024-03-04 21:23:45 -05:00
parent 1b342d56ef
commit a7790a271f
7 changed files with 7 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELIST.md
View File

@@ -10,6 +10,7 @@
- Hide size if value is 0 (Deterous)
- Fix title normalization (Deterous)
- Ensure no labels are empty
- Use SabreTools.Hashing
### 3.1.2 (2024-02-27)

350
MPF.Core/Hashing/Hasher.cs
View File

@@ -1,350 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
#if NET6_0_OR_GREATER
using System.IO.Hashing;
#endif
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using MPF.Core.Data;
namespace MPF.Core.Hashing
{
public sealed class Hasher : IDisposable
{
#region Properties
/// <summary>
/// Hash type associated with the current state
/// </summary>
#if NETFRAMEWORK || NETCOREAPP3_1
public Hash HashType { get; private set; }
#else
public Hash HashType { get; init; }
#endif
/// <summary>
/// Current hash in bytes
/// </summary>
public byte[]? CurrentHashBytes
{
get
{
return (_hasher) switch
{
HashAlgorithm ha => ha.Hash,
NonCryptographicHashAlgorithm ncha => ncha.GetCurrentHash().Reverse().ToArray(),
_ => null,
};
}
}
/// <summary>
/// Current hash as a string
/// </summary>
public string? CurrentHashString => ByteArrayToString(CurrentHashBytes);
#endregion
#region Private Fields
/// <summary>
/// Internal hasher being used for processing
/// </summary>
/// <remarks>May be either a HashAlgorithm or NonCryptographicHashAlgorithm</remarks>
private object? _hasher;
#endregion
#region Constructors
/// <summary>
/// Constructor
/// </summary>
/// <param name="hashType">Hash type to instantiate</param>
public Hasher(Hash hashType)
{
this.HashType = hashType;
GetHasher();
}
/// <summary>
/// Generate the correct hashing class based on the hash type
/// </summary>
private void GetHasher()
{
_hasher = HashType switch
{
Hash.CRC32 => new Crc32(),
#if NET6_0_OR_GREATER
Hash.CRC64 => new Crc64(),
#endif
Hash.MD5 => MD5.Create(),
Hash.SHA1 => SHA1.Create(),
Hash.SHA256 => SHA256.Create(),
Hash.SHA384 => SHA384.Create(),
Hash.SHA512 => SHA512.Create(),
#if NET6_0_OR_GREATER
Hash.XxHash32 => new XxHash32(),
Hash.XxHash64 => new XxHash64(),
#endif
_ => null,
};
}
/// <inheritdoc/>
public void Dispose()
{
if (_hasher is IDisposable disposable)
disposable.Dispose();
}
#endregion
#region Static Hashing
/// <summary>
/// Get hashes from an input file path
/// </summary>
/// <param name="filename">Path to the input file</param>
/// <returns>True if hashing was successful, false otherwise</returns>
public static bool GetFileHashes(string filename, out long size, out string? crc32, out string? md5, out string? sha1)
{
// Set all initial values
crc32 = null; md5 = null; sha1 = null;
// Get all file hashes
var fileHashes = GetFileHashes(filename, out size);
if (fileHashes == null)
return false;
// Assign the file hashes and return
crc32 = fileHashes[Hash.CRC32];
md5 = fileHashes[Hash.MD5];
sha1 = fileHashes[Hash.SHA1];
return true;
}
/// <summary>
/// Get hashes from an input file path
/// </summary>
/// <param name="filename">Path to the input file</param>
/// <returns>Dictionary containing hashes on success, null on error</returns>
public static Dictionary<Hash, string?>? GetFileHashes(string filename, out long size)
{
// If the file doesn't exist, we can't do anything
if (!File.Exists(filename))
{
size = -1;
return null;
}
// Set the file size
size = new FileInfo(filename).Length;
// Open the input file
var input = File.OpenRead(filename);
// Return the hashes from the stream
return GetStreamHashes(input);
}
/// <summary>
/// Get hashes from an input Stream
/// </summary>
/// <param name="input">Stream to hash</param>
/// <returns>Dictionary containing hashes on success, null on error</returns>
public static Dictionary<Hash, string?>? GetStreamHashes(Stream input)
{
// Create the output dictionary
var hashDict = new Dictionary<Hash, string?>();
try
{
// Get a list of hashers to run over the buffer
var hashers = new Dictionary<Hash, Hasher>
{
{ Hash.CRC32, new Hasher(Hash.CRC32) },
#if NET6_0_OR_GREATER
{ Hash.CRC64, new Hasher(Hash.CRC64) },
#endif
{ Hash.MD5, new Hasher(Hash.MD5) },
{ Hash.SHA1, new Hasher(Hash.SHA1) },
{ Hash.SHA256, new Hasher(Hash.SHA256) },
{ Hash.SHA384, new Hasher(Hash.SHA384) },
{ Hash.SHA512, new Hasher(Hash.SHA512) },
#if NET6_0_OR_GREATER
{ Hash.XxHash32, new Hasher(Hash.XxHash32) },
{ Hash.XxHash64, new Hasher(Hash.XxHash64) },
#endif
};
// Initialize the hashing helpers
var loadBuffer = new ThreadLoadBuffer(input);
int buffersize = 3 * 1024 * 1024;
byte[] buffer0 = new byte[buffersize];
byte[] buffer1 = new byte[buffersize];
/*
Please note that some of the following code is adapted from
RomVault. This is a modified version of how RomVault does
threaded hashing. As such, some of the terminology and code
is the same, though variable names and comments may have
been tweaked to better fit this code base.
*/
// Pre load the first buffer
long refsize = input.Length;
int next = refsize > buffersize ? buffersize : (int)refsize;
input.Read(buffer0, 0, next);
int current = next;
refsize -= next;
bool bufferSelect = true;
while (current > 0)
{
// Trigger the buffer load on the second buffer
next = refsize > buffersize ? buffersize : (int)refsize;
if (next > 0)
loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next);
byte[] buffer = bufferSelect ? buffer0 : buffer1;
#if NET20 || NET35
// Run hashers sequentially on each chunk
foreach (var h in hashers)
{
h.Value.Process(buffer, current);
}
#else
// Run hashers in parallel on each chunk
Parallel.ForEach(hashers, h => h.Value.Process(buffer, current));
#endif
// Wait for the load buffer worker, if needed
if (next > 0)
loadBuffer.Wait();
// Setup for the next hashing step
current = next;
refsize -= next;
bufferSelect = !bufferSelect;
}
// Finalize all hashing helpers
loadBuffer.Finish();
#if NET20 || NET35
foreach (var h in hashers)
{
h.Value.Terminate();
}
#else
Parallel.ForEach(hashers, h => h.Value.Terminate());
#endif
// Get the results
hashDict[Hash.CRC32] = hashers[Hash.CRC32].CurrentHashString;
#if NET6_0_OR_GREATER
hashDict[Hash.CRC64] = hashers[Hash.CRC64].CurrentHashString;
#endif
hashDict[Hash.MD5] = hashers[Hash.MD5].CurrentHashString;
hashDict[Hash.SHA1] = hashers[Hash.SHA1].CurrentHashString;
hashDict[Hash.SHA256] = hashers[Hash.SHA256].CurrentHashString;
hashDict[Hash.SHA384] = hashers[Hash.SHA384].CurrentHashString;
hashDict[Hash.SHA512] = hashers[Hash.SHA512].CurrentHashString;
#if NET6_0_OR_GREATER
hashDict[Hash.XxHash32] = hashers[Hash.XxHash32].CurrentHashString;
hashDict[Hash.XxHash64] = hashers[Hash.XxHash64].CurrentHashString;
#endif
// Dispose of the hashers
loadBuffer.Dispose();
foreach (var hasher in hashers.Values)
{
hasher.Dispose();
}
return hashDict;
}
catch (IOException)
{
return null;
}
finally
{
input.Dispose();
}
}
#endregion
#region Hashing
/// <summary>
/// Process a buffer of some length with the internal hash algorithm
/// </summary>
public void Process(byte[] buffer, int size)
{
switch (_hasher)
{
case HashAlgorithm ha:
ha.TransformBlock(buffer, 0, size, null, 0);
break;
case NonCryptographicHashAlgorithm ncha:
#if NET20 || NET35 || NET40
byte[] bufferSpan = new byte[size];
Array.Copy(buffer, bufferSpan, size);
#else
var bufferSpan = new ReadOnlySpan<byte>(buffer, 0, size);
#endif
ncha.Append(bufferSpan);
break;
}
}
/// <summary>
/// Finalize the internal hash algorigthm
/// </summary>
/// <remarks>NonCryptographicHashAlgorithm implementations do not need finalization</remarks>
public void Terminate()
{
byte[] emptyBuffer = [];
switch (_hasher)
{
case HashAlgorithm ha:
ha.TransformFinalBlock(emptyBuffer, 0, 0);
break;
}
}
#endregion
#region Helpers
/// <summary>
/// Convert a byte array to a hex string
/// </summary>
/// <param name="bytes">Byte array to convert</param>
/// <returns>Hex string representing the byte array</returns>
/// <link>http://stackoverflow.com/questions/311165/how-do-you-convert-byte-array-to-hexadecimal-string-and-vice-versa</link>
private static string? ByteArrayToString(byte[]? bytes)
{
// If we get null in, we send null out
if (bytes == null)
return null;
try
{
string hex = BitConverter.ToString(bytes);
return hex.Replace("-", string.Empty).ToLowerInvariant();
}
catch
{
return null;
}
}
#endregion
}
}

184
MPF.Core/Hashing/OptimizedCRC.cs
View File

@@ -1,184 +0,0 @@
#if NETFRAMEWORK || NETCOREAPP3_1 || NET5_0
/*
Copyright (c) 2012-2015 Eugene Larchenko (spct@mail.ru)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
using System;
//namespace OptimizedCRC
namespace MPF.Core.Hashing
{
/// <summary>
/// Shell class to trick older versions into using CRC-32 properly
/// </summary>
internal abstract class NonCryptographicHashAlgorithm
{
#if NET20 || NET35 || NET40
/// <summary>
/// When overridden in a derived class, appends the contents of source to
/// the data already processed for the current hash computation.
/// </summary>
/// <param name="source">The data to process.</param>
public abstract void Append(byte[] source);
#else
/// <summary>
/// When overridden in a derived class, appends the contents of source to
/// the data already processed for the current hash computation.
/// </summary>
/// <param name="source">The data to process.</param>
public abstract void Append(ReadOnlySpan<byte> source);
#endif
/// <summary>
/// Gets the current computed hash value without modifying accumulated state.
/// </summary>
/// <returns>The hash value for the data already provided.</returns>
public abstract byte[] GetCurrentHash();
}
/// <remarks>
/// Some changes have been made to this code to make it more similar to the System.IO.Hashing implementations
/// </remarks>
internal class Crc32 : NonCryptographicHashAlgorithm, IDisposable
{
private const uint kCrcPoly = 0xEDB88320;
private const uint kInitial = 0xFFFFFFFF;
private const int CRC_NUM_TABLES = 8;
private static readonly uint[] Table;
static Crc32()
{
unchecked
{
Table = new uint[256 * CRC_NUM_TABLES];
int i;
for (i = 0; i < 256; i++)
{
uint r = (uint)i;
for (int j = 0; j < 8; j++)
{
r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1));
}
Table[i] = r;
}
for (; i < 256 * CRC_NUM_TABLES; i++)
{
uint r = Table[i - 256];
Table[i] = Table[r & 0xFF] ^ (r >> 8);
}
}
}
public uint UnsignedValue;
public Crc32()
{
Init();
}
/// <summary>
/// Reset CRC
/// </summary>
public void Init()
{
UnsignedValue = kInitial;
}
/// <inheritdoc/>
public override byte[] GetCurrentHash()
{
return BitConverter.GetBytes(~UnsignedValue);
}
/// <inheritdoc/>
#if NET20 || NET35 || NET40
public override void Append(byte[] source)
{
Update(source, 0, source.Length);
}
#else
public override void Append(ReadOnlySpan<byte> source)
{
byte[] sourceBytes = source.ToArray();
Update(sourceBytes, 0, sourceBytes.Length);
}
#endif
private void Update(byte[] data, int offset, int count)
{
_ = new ArraySegment<byte>(data, offset, count); // check arguments
if (count == 0)
{
return;
}
var table = Table;
uint crc = UnsignedValue;
for (; (offset & 7) != 0 && count != 0; count--)
{
crc = (crc >> 8) ^ table[(byte)crc ^ data[offset++]];
}
if (count >= 8)
{
/*
* Idea from 7-zip project sources (http://7-zip.org/sdk.html)
*/
int end = (count - 8) & ~7;
count -= end;
end += offset;
while (offset != end)
{
crc ^= (uint)(data[offset] + (data[offset + 1] << 8) + (data[offset + 2] << 16) + (data[offset + 3] << 24));
uint high = (uint)(data[offset + 4] + (data[offset + 5] << 8) + (data[offset + 6] << 16) + (data[offset + 7] << 24));
offset += 8;
crc = table[(byte)crc + 0x700]
^ table[(byte)(crc >>= 8) + 0x600]
^ table[(byte)(crc >>= 8) + 0x500]
^ table[/*(byte)*/(crc >> 8) + 0x400]
^ table[(byte)(high) + 0x300]
^ table[(byte)(high >>= 8) + 0x200]
^ table[(byte)(high >>= 8) + 0x100]
^ table[/*(byte)*/(high >> 8) + 0x000];
}
}
while (count-- != 0)
{
crc = (crc >> 8) ^ table[(byte)crc ^ data[offset++]];
}
UnsignedValue = crc;
}
public void Dispose()
{
UnsignedValue = 0;
}
}
}
#endif

81
MPF.Core/Hashing/ThreadLoadBuffer.cs
View File

@@ -1,81 +0,0 @@
using System;
using System.IO;
using System.Threading;
//namespace Compress.ThreadReaders
namespace MPF.Core.Hashing
{
public sealed class ThreadLoadBuffer : IDisposable
{
private readonly AutoResetEvent _waitEvent;
private readonly AutoResetEvent _outEvent;
private readonly Thread _tWorker;
private byte[]? _buffer;
private int _size;
private readonly Stream _ds;
private bool _finished;
public bool errorState;
public int SizeRead;
public ThreadLoadBuffer(Stream ds)
{
_waitEvent = new AutoResetEvent(false);
_outEvent = new AutoResetEvent(false);
_finished = false;
_ds = ds;
errorState = false;
_tWorker = new Thread(MainLoop);
_tWorker.Start();
}
public void Dispose()
{
_waitEvent.Close();
_outEvent.Close();
}
private void MainLoop()
{
while (true)
{
_waitEvent.WaitOne();
if (_finished)
{
break;
}
try
{
if (_buffer != null)
SizeRead = _ds.Read(_buffer, 0, _size);
}
catch (Exception)
{
errorState = true;
}
_outEvent.Set();
}
}
public void Trigger(byte[] buffer, int size)
{
_buffer = buffer;
_size = size;
_waitEvent.Set();
}
public void Wait()
{
_outEvent.WaitOne();
}
public void Finish()
{
_finished = true;
_waitEvent.Set();
_tWorker.Join();
}
}
}

1
MPF.Core/MPF.Core.csproj
View File

@@ -56,6 +56,7 @@
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="psxt001z.Library" Version="0.21.0-beta3" />
<PackageReference Include="SabreTools.Hashing" Version="1.1.2" />
<PackageReference Include="SabreTools.Models" Version="1.3.0" />
<PackageReference Include="SabreTools.RedumpLib" Version="1.3.4" />
<PackageReference Include="SabreTools.Serialization" Version="1.3.2" />

5
MPF.Core/Modules/PS3CFW/Parameters.cs
View File

@@ -5,6 +5,7 @@ using System.Text;
using System.Text.RegularExpressions;
using MPF.Core.Converters;
using MPF.Core.Data;
using SabreTools.Hashing;
using SabreTools.RedumpLib;
using SabreTools.RedumpLib.Data;
@@ -171,7 +172,7 @@ namespace MPF.Core.Modules.PS3CFW
try
{
if (Hashing.Hasher.GetFileHashes(iso, out long size, out string? crc, out string? md5, out string? sha1))
if (HashTool.GetStandardHashes(iso, out long size, out string? crc, out string? md5, out string? sha1))
{
return new Datafile
{
@@ -254,7 +255,7 @@ namespace MPF.Core.Modules.PS3CFW
try
{
if (Hashing.Hasher.GetFileHashes(iso, out long size, out string? crc, out string? md5, out string? sha1))
if (HashTool.GetStandardHashes(iso, out long size, out string? crc, out string? md5, out string? sha1))
return $"<rom name=\"{Path.GetFileName(iso)}\" size=\"{size}\" crc=\"{crc}\" md5=\"{md5}\" sha1=\"{sha1}\" />";
return null;
}

4
MPF.Core/Modules/UmdImageCreator/Parameters.cs
View File

@@ -4,7 +4,7 @@ using System.IO;
using System.Linq;
using MPF.Core.Converters;
using MPF.Core.Data;
using MPF.Core.Hashing;
using SabreTools.Hashing;
using SabreTools.RedumpLib;
using SabreTools.RedumpLib.Data;
@@ -78,7 +78,7 @@ namespace MPF.Core.Modules.UmdImageCreator
case MediaType.UMD:
info.Extras!.PVD = GetPVD(basePath + "_mainInfo.txt") ?? string.Empty;
if (Hasher.GetFileHashes(basePath + ".iso", out long filesize, out var crc32, out var md5, out var sha1))
if (HashTool.GetStandardHashes(basePath + ".iso", out long filesize, out var crc32, out var md5, out var sha1))
{
// Get the Datafile information
var datafile = new Datafile