From 25dd3f4921e6651c3313ec7b3e2fcc21150e192f Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sat, 22 Aug 2020 05:14:52 +0100 Subject: [PATCH] Hash and compress DAT file on import. --- .../Checksums/BigEndianBitConverter.cs | 320 +++++++++++ RomRepoMgr.Core/Checksums/CRC32Context.cs | 233 ++++++++ RomRepoMgr.Core/Checksums/CRC64Context.cs | 235 ++++++++ RomRepoMgr.Core/Checksums/IChecksum.cs | 58 ++ RomRepoMgr.Core/Checksums/MD5Context.cs | 132 +++++ RomRepoMgr.Core/Checksums/SHA1Context.cs | 132 +++++ RomRepoMgr.Core/Checksums/SHA256Context.cs | 132 +++++ RomRepoMgr.Core/Checksums/SHA384Context.cs | 132 +++++ RomRepoMgr.Core/Checksums/SHA512Context.cs | 132 +++++ RomRepoMgr.Core/Checksums/SpamSumContext.cs | 526 ++++++++++++++++++ RomRepoMgr.Core/RomRepoMgr.Core.csproj | 5 +- RomRepoMgr.Core/Workers/Compression.cs | 57 ++ RomRepoMgr.Core/Workers/DatImporter.cs | 42 +- .../Interop/DetectOS.cs | 0 .../Interop/PlatformID.cs | 0 .../Interop/Version.cs | 0 .../RomRepoMgr.Settings.csproj | 4 - 17 files changed, 2134 insertions(+), 6 deletions(-) create mode 100644 RomRepoMgr.Core/Checksums/BigEndianBitConverter.cs create mode 100644 RomRepoMgr.Core/Checksums/CRC32Context.cs create mode 100644 RomRepoMgr.Core/Checksums/CRC64Context.cs create mode 100644 RomRepoMgr.Core/Checksums/IChecksum.cs create mode 100644 RomRepoMgr.Core/Checksums/MD5Context.cs create mode 100644 RomRepoMgr.Core/Checksums/SHA1Context.cs create mode 100644 RomRepoMgr.Core/Checksums/SHA256Context.cs create mode 100644 RomRepoMgr.Core/Checksums/SHA384Context.cs create mode 100644 RomRepoMgr.Core/Checksums/SHA512Context.cs create mode 100644 RomRepoMgr.Core/Checksums/SpamSumContext.cs create mode 100644 RomRepoMgr.Core/Workers/Compression.cs rename {RomRepoMgr.Core => RomRepoMgr.Settings}/Interop/DetectOS.cs (100%) rename {RomRepoMgr.Core => RomRepoMgr.Settings}/Interop/PlatformID.cs (100%) rename {RomRepoMgr.Core => RomRepoMgr.Settings}/Interop/Version.cs (100%) diff --git a/RomRepoMgr.Core/Checksums/BigEndianBitConverter.cs b/RomRepoMgr.Core/Checksums/BigEndianBitConverter.cs new file mode 100644 index 0000000..a4179c3 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/BigEndianBitConverter.cs @@ -0,0 +1,320 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : BigEndianBitConverter.cs +// Author(s) : Natalia Portillo +// +// Component : Helpers. +// +// --[ Description ] ---------------------------------------------------------- +// +// Override of System.BitConverter that knows how to handle big-endian. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System; +using System.Linq; + +namespace Aaru.Helpers +{ + /// + /// Converts base data types to an array of bytes, and an array of bytes to base data types. All info taken from + /// the meta data of System.BitConverter. This implementation allows for Endianness consideration. + /// + public static class BigEndianBitConverter + { + /// Converts the specified double-precision floating point number to a 64-bit signed integer. + /// The number to convert. + /// A 64-bit signed integer whose value is equivalent to value. + /// It is not currently implemented + public static long DoubleToInt64Bits(double value) => throw new NotImplementedException(); + + /// Returns the specified Boolean value as an array of bytes. + /// A Boolean value. + /// An array of bytes with length 1. + public static byte[] GetBytes(bool value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified Unicode character value as an array of bytes. + /// A character to convert. + /// An array of bytes with length 2. + public static byte[] GetBytes(char value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified double-precision floating point value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 8. + public static byte[] GetBytes(double value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified single-precision floating point value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 4. + public static byte[] GetBytes(float value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 32-bit signed integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 4. + public static byte[] GetBytes(int value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 64-bit signed integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 8. + public static byte[] GetBytes(long value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 16-bit signed integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 2. + public static byte[] GetBytes(short value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 32-bit unsigned integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 4. + public static byte[] GetBytes(uint value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 64-bit unsigned integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 8. + public static byte[] GetBytes(ulong value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Returns the specified 16-bit unsigned integer value as an array of bytes. + /// The number to convert. + /// An array of bytes with length 2. + public static byte[] GetBytes(ushort value) => BitConverter.GetBytes(value).Reverse().ToArray(); + + /// Converts the specified 64-bit signed integer to a double-precision floating point number. + /// The number to convert. + /// A double-precision floating point number whose value is equivalent to value. + public static double Int64BitsToDouble(long value) => throw new NotImplementedException(); + + /// Returns a Boolean value converted from one byte at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// true if the byte at in value is nonzero; otherwise, false. + /// value is null. + /// + /// is less than zero or greater than the + /// length of value minus 1. + /// + public static bool ToBoolean(byte[] value, int startIndex) => throw new NotImplementedException(); + + /// Returns a Unicode character converted from two bytes at a specified position in a byte array. + /// An array. + /// The starting position within value. + /// A character formed by two bytes beginning at . + /// equals the length of value minus 1. + /// value is null. + /// + /// is less than zero or greater than the + /// length of value minus 1. + /// + public static char ToChar(byte[] value, int startIndex) => throw new NotImplementedException(); + + /// + /// Returns a double-precision floating point number converted from eight bytes at a specified position in a byte + /// array. + /// + /// An array of bytes. + /// The starting position within value. + /// A double precision floating point number formed by eight bytes beginning at . + /// + /// is greater than or equal to the length of value + /// minus 7, and is less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// is less than zero or greater than the + /// length of value minus 1. + /// + public static double ToDouble(byte[] value, int startIndex) => throw new NotImplementedException(); + + /// Returns a 16-bit signed integer converted from two bytes at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// A 16-bit signed integer formed by two bytes beginning at . + /// equals the length of value minus 1. + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static short ToInt16(byte[] value, int startIndex) => + BitConverter.ToInt16(value.Reverse().ToArray(), value.Length - sizeof(short) - startIndex); + + /// Returns a 32-bit signed integer converted from four bytes at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// A 32-bit signed integer formed by four bytes beginning at . + /// + /// is greater than or equal to the length of value + /// minus 3, and is less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static int ToInt32(byte[] value, int startIndex) => + BitConverter.ToInt32(value.Reverse().ToArray(), value.Length - sizeof(int) - startIndex); + + /// Returns a 64-bit signed integer converted from eight bytes at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// A 64-bit signed integer formed by eight bytes beginning at . + /// + /// is greater than or equal to the length of value + /// minus 7, and is less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// is less than zero or greater than the + /// length of value minus 1. + /// + public static long ToInt64(byte[] value, int startIndex) => + BitConverter.ToInt64(value.Reverse().ToArray(), value.Length - sizeof(long) - startIndex); + + /// + /// Returns a single-precision floating point number converted from four bytes at a specified position in a byte + /// array. + /// + /// An array of bytes. + /// The starting position within value. + /// A single-precision floating point number formed by four bytes beginning at . + /// + /// is greater than or equal to the length of value + /// minus 3, and is less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// is less than zero or greater than the + /// length of value minus 1. + /// + public static float ToSingle(byte[] value, int startIndex) => + BitConverter.ToSingle(value.Reverse().ToArray(), value.Length - sizeof(float) - startIndex); + + /// + /// Converts the numeric value of each element of a specified array of bytes to its equivalent hexadecimal string + /// representation. + /// + /// An array of bytes. + /// + /// A System.String of hexadecimal pairs separated by hyphens, where each pair represents the corresponding + /// element in value; for example, "7F-2C-4A". + /// + /// value is null. + public static string ToString(byte[] value) => BitConverter.ToString(value.Reverse().ToArray()); + + /// + /// Converts the numeric value of each element of a specified subarray of bytes to its equivalent hexadecimal + /// string representation. + /// + /// An array of bytes. + /// The starting position within value. + /// + /// A System.String of hexadecimal pairs separated by hyphens, where each pair represents the corresponding + /// element in a subarray of value; for example, "7F-2C-4A". + /// + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static string ToString(byte[] value, int startIndex) => + BitConverter.ToString(value.Reverse().ToArray(), startIndex); + + /// + /// Converts the numeric value of each element of a specified subarray of bytes to its equivalent hexadecimal + /// string representation. + /// + /// An array of bytes. + /// The starting position within value. + /// The number of array elements in value to convert. + /// + /// A System.String of hexadecimal pairs separated by hyphens, where each pair represents the corresponding + /// element in a subarray of value; for example, "7F-2C-4A". + /// + /// value is null. + /// + /// startIndex or length is less than zero. -or- startIndex is greater + /// than zero and is greater than or equal to the length of value. + /// + /// + /// The combination of startIndex and length does not specify a position within + /// value; that is, the startIndex parameter is greater than the length of value minus the length parameter. + /// + public static string ToString(byte[] value, int startIndex, int length) => + BitConverter.ToString(value.Reverse().ToArray(), startIndex, length); + + /// Returns a 16-bit unsigned integer converted from two bytes at a specified position in a byte array. + /// The array of bytes. + /// The starting position within value. + /// A 16-bit unsigned integer formed by two bytes beginning at startIndex. + /// startIndex equals the length of value minus 1. + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static ushort ToUInt16(byte[] value, int startIndex) => + BitConverter.ToUInt16(value.Reverse().ToArray(), value.Length - sizeof(ushort) - startIndex); + + /// Returns a 32-bit unsigned integer converted from four bytes at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// A 32-bit unsigned integer formed by four bytes beginning at startIndex. + /// + /// startIndex is greater than or equal to the length of value minus 3, and is + /// less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static uint ToUInt32(byte[] value, int startIndex) => + BitConverter.ToUInt32(value.Reverse().ToArray(), value.Length - sizeof(uint) - startIndex); + + /// Returns a 64-bit unsigned integer converted from eight bytes at a specified position in a byte array. + /// An array of bytes. + /// The starting position within value. + /// A 64-bit unsigned integer formed by the eight bytes beginning at startIndex. + /// + /// startIndex is greater than or equal to the length of value minus 7, and is + /// less than or equal to the length of value minus 1. + /// + /// value is null. + /// + /// startIndex is less than zero or greater than the length of value + /// minus 1. + /// + public static ulong ToUInt64(byte[] value, int startIndex) => + BitConverter.ToUInt64(value.Reverse().ToArray(), value.Length - sizeof(ulong) - startIndex); + + public static Guid ToGuid(byte[] value, int startIndex) => new Guid(ToUInt32(value, 0 + startIndex), + ToUInt16(value, 4 + startIndex), + ToUInt16(value, 6 + startIndex), + value[8 + startIndex + 0], + value[8 + startIndex + 1], + value[8 + startIndex + 2], + value[8 + startIndex + 3], + value[8 + startIndex + 5], + value[8 + startIndex + 5], + value[8 + startIndex + 6], + value[8 + startIndex + 7]); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/CRC32Context.cs b/RomRepoMgr.Core/Checksums/CRC32Context.cs new file mode 100644 index 0000000..e4c3745 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/CRC32Context.cs @@ -0,0 +1,233 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : CRC32Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Implements a CRC32 algorithm. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Text; +using Aaru.CommonTypes.Interfaces; +using Aaru.Helpers; + +namespace Aaru.Checksums +{ + /// Implements a CRC32 algorithm + public sealed class Crc32Context : IChecksum + { + const uint CRC32_ISO_POLY = 0xEDB88320; + const uint CRC32_ISO_SEED = 0xFFFFFFFF; + + readonly uint _finalSeed; + readonly uint[] _table; + uint _hashInt; + + /// Initializes the CRC32 table and seed as CRC32-ISO + public Crc32Context() + { + _hashInt = CRC32_ISO_SEED; + _finalSeed = CRC32_ISO_SEED; + + _table = new uint[256]; + + for(int i = 0; i < 256; i++) + { + uint entry = (uint)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ CRC32_ISO_POLY; + else + entry = entry >> 1; + + _table[i] = entry; + } + } + + /// Initializes the CRC32 table with a custom polynomial and seed + public Crc32Context(uint polynomial, uint seed) + { + _hashInt = seed; + _finalSeed = seed; + + _table = new uint[256]; + + for(int i = 0; i < 256; i++) + { + uint entry = (uint)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + _table[i] = entry; + } + } + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) + { + for(int i = 0; i < len; i++) + _hashInt = (_hashInt >> 8) ^ _table[data[i] ^ (_hashInt & 0xff)]; + } + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() => BigEndianBitConverter.GetBytes(_hashInt ^ _finalSeed); + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + var crc32Output = new StringBuilder(); + + for(int i = 0; i < BigEndianBitConverter.GetBytes(_hashInt ^ _finalSeed).Length; i++) + crc32Output.Append(BigEndianBitConverter.GetBytes(_hashInt ^ _finalSeed)[i].ToString("x2")); + + return crc32Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + File(filename, out byte[] hash); + + return hash; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) => + File(filename, out hash, CRC32_ISO_POLY, CRC32_ISO_SEED); + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + /// CRC polynomial + /// CRC seed + public static string File(string filename, out byte[] hash, uint polynomial, uint seed) + { + var fileStream = new FileStream(filename, FileMode.Open); + + uint localhashInt = seed; + + uint[] localTable = new uint[256]; + + for(int i = 0; i < 256; i++) + { + uint entry = (uint)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + localTable[i] = entry; + } + + for(int i = 0; i < fileStream.Length; i++) + localhashInt = (localhashInt >> 8) ^ localTable[fileStream.ReadByte() ^ (localhashInt & 0xff)]; + + localhashInt ^= seed; + hash = BigEndianBitConverter.GetBytes(localhashInt); + + var crc32Output = new StringBuilder(); + + foreach(byte h in hash) + crc32Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return crc32Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) => + Data(data, len, out hash, CRC32_ISO_POLY, CRC32_ISO_SEED); + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + /// CRC polynomial + /// CRC seed + public static string Data(byte[] data, uint len, out byte[] hash, uint polynomial, uint seed) + { + uint localhashInt = seed; + + uint[] localTable = new uint[256]; + + for(int i = 0; i < 256; i++) + { + uint entry = (uint)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + localTable[i] = entry; + } + + for(int i = 0; i < len; i++) + localhashInt = (localhashInt >> 8) ^ localTable[data[i] ^ (localhashInt & 0xff)]; + + localhashInt ^= seed; + hash = BigEndianBitConverter.GetBytes(localhashInt); + + var crc32Output = new StringBuilder(); + + foreach(byte h in hash) + crc32Output.Append(h.ToString("x2")); + + return crc32Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/CRC64Context.cs b/RomRepoMgr.Core/Checksums/CRC64Context.cs new file mode 100644 index 0000000..d9bc026 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/CRC64Context.cs @@ -0,0 +1,235 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : CRC64Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Implements a CRC64 algorithm. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Text; +using Aaru.CommonTypes.Interfaces; +using Aaru.Helpers; + +namespace Aaru.Checksums +{ + /// Implements a CRC64 algorithm + public sealed class Crc64Context : IChecksum + { + public const ulong CRC64_ECMA_POLY = 0xC96C5795D7870F42; + public const ulong CRC64_ECMA_SEED = 0xFFFFFFFFFFFFFFFF; + + readonly ulong _finalSeed; + readonly ulong[] _table; + ulong _hashInt; + + /// Initializes the CRC64 table and seed as CRC64-ECMA + public Crc64Context() + { + _hashInt = CRC64_ECMA_SEED; + + _table = new ulong[256]; + + for(int i = 0; i < 256; i++) + { + ulong entry = (ulong)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ CRC64_ECMA_POLY; + else + entry = entry >> 1; + + _table[i] = entry; + } + + _finalSeed = CRC64_ECMA_SEED; + } + + /// Initializes the CRC16 table with a custom polynomial and seed + public Crc64Context(ulong polynomial, ulong seed) + { + _hashInt = seed; + + _table = new ulong[256]; + + for(int i = 0; i < 256; i++) + { + ulong entry = (ulong)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + _table[i] = entry; + } + + _finalSeed = seed; + } + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) + { + for(int i = 0; i < len; i++) + _hashInt = (_hashInt >> 8) ^ _table[data[i] ^ (_hashInt & 0xff)]; + } + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() => BigEndianBitConverter.GetBytes(_hashInt ^= _finalSeed); + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + var crc64Output = new StringBuilder(); + + for(int i = 0; i < BigEndianBitConverter.GetBytes(_hashInt ^= _finalSeed).Length; i++) + crc64Output.Append(BigEndianBitConverter.GetBytes(_hashInt ^= _finalSeed)[i].ToString("x2")); + + return crc64Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + File(filename, out byte[] localHash); + + return localHash; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) => + File(filename, out hash, CRC64_ECMA_POLY, CRC64_ECMA_SEED); + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + /// CRC polynomial + /// CRC seed + public static string File(string filename, out byte[] hash, ulong polynomial, ulong seed) + { + var fileStream = new FileStream(filename, FileMode.Open); + + ulong localhashInt = seed; + + ulong[] localTable = new ulong[256]; + + for(int i = 0; i < 256; i++) + { + ulong entry = (ulong)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + localTable[i] = entry; + } + + for(int i = 0; i < fileStream.Length; i++) + localhashInt = (localhashInt >> 8) ^ localTable[(ulong)fileStream.ReadByte() ^ (localhashInt & 0xffL)]; + + localhashInt ^= seed; + hash = BigEndianBitConverter.GetBytes(localhashInt); + + var crc64Output = new StringBuilder(); + + foreach(byte h in hash) + crc64Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return crc64Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) => + Data(data, len, out hash, CRC64_ECMA_POLY, CRC64_ECMA_SEED); + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + /// CRC polynomial + /// CRC seed + public static string Data(byte[] data, uint len, out byte[] hash, ulong polynomial, ulong seed) + { + ulong localhashInt = seed; + + ulong[] localTable = new ulong[256]; + + for(int i = 0; i < 256; i++) + { + ulong entry = (ulong)i; + + for(int j = 0; j < 8; j++) + if((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry = entry >> 1; + + localTable[i] = entry; + } + + for(int i = 0; i < len; i++) + localhashInt = (localhashInt >> 8) ^ localTable[data[i] ^ (localhashInt & 0xff)]; + + localhashInt ^= seed; + hash = BigEndianBitConverter.GetBytes(localhashInt); + + var crc64Output = new StringBuilder(); + + foreach(byte h in hash) + crc64Output.Append(h.ToString("x2")); + + return crc64Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/IChecksum.cs b/RomRepoMgr.Core/Checksums/IChecksum.cs new file mode 100644 index 0000000..cb957e0 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/IChecksum.cs @@ -0,0 +1,58 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : IChecksum.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Provides an interface for implementing checksums and hashes. +// +// --[ License ] -------------------------------------------------------------- +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +namespace Aaru.CommonTypes.Interfaces +{ + public interface IChecksum + { + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + void Update(byte[] data, uint len); + + /// Updates the hash with data. + /// Data buffer. + void Update(byte[] data); + + /// Returns a byte array of the hash value. + byte[] Final(); + + /// Returns a hexadecimal representation of the hash value. + string End(); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/MD5Context.cs b/RomRepoMgr.Core/Checksums/MD5Context.cs new file mode 100644 index 0000000..95ff00b --- /dev/null +++ b/RomRepoMgr.Core/Checksums/MD5Context.cs @@ -0,0 +1,132 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : MD5Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Wraps up .NET MD5 implementation to a Init(), Update(), Final() context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Wraps up .NET MD5 implementation to a Init(), Update(), Final() context. + public sealed class Md5Context : IChecksum + { + readonly MD5 _provider; + + /// Initializes the MD5 hash provider + public Md5Context() => _provider = MD5.Create(); + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) => _provider.TransformBlock(data, 0, (int)len, data, 0); + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + + return _provider.Hash; + } + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + var md5Output = new StringBuilder(); + + foreach(byte h in _provider.Hash) + md5Output.Append(h.ToString("x2")); + + return md5Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + var localMd5Provider = MD5.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + byte[] result = localMd5Provider.ComputeHash(fileStream); + fileStream.Close(); + + return result; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) + { + var localMd5Provider = MD5.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + hash = localMd5Provider.ComputeHash(fileStream); + var md5Output = new StringBuilder(); + + foreach(byte h in hash) + md5Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return md5Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) + { + var localMd5Provider = MD5.Create(); + hash = localMd5Provider.ComputeHash(data, 0, (int)len); + var md5Output = new StringBuilder(); + + foreach(byte h in hash) + md5Output.Append(h.ToString("x2")); + + return md5Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/SHA1Context.cs b/RomRepoMgr.Core/Checksums/SHA1Context.cs new file mode 100644 index 0000000..a054b2a --- /dev/null +++ b/RomRepoMgr.Core/Checksums/SHA1Context.cs @@ -0,0 +1,132 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : SHA1Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Wraps up .NET SHA1 implementation to a Init(), Update(), Final() context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Wraps up .NET SHA1 implementation to a Init(), Update(), Final() context. + public sealed class Sha1Context : IChecksum + { + readonly SHA1 _provider; + + /// Initializes the SHA1 hash provider + public Sha1Context() => _provider = SHA1.Create(); + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) => _provider.TransformBlock(data, 0, (int)len, data, 0); + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + + return _provider.Hash; + } + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + var sha1Output = new StringBuilder(); + + foreach(byte h in _provider.Hash) + sha1Output.Append(h.ToString("x2")); + + return sha1Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + var localSha1Provider = SHA1.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + byte[] result = localSha1Provider.ComputeHash(fileStream); + fileStream.Close(); + + return result; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) + { + var localSha1Provider = SHA1.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + hash = localSha1Provider.ComputeHash(fileStream); + var sha1Output = new StringBuilder(); + + foreach(byte h in hash) + sha1Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return sha1Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) + { + var localSha1Provider = SHA1.Create(); + hash = localSha1Provider.ComputeHash(data, 0, (int)len); + var sha1Output = new StringBuilder(); + + foreach(byte h in hash) + sha1Output.Append(h.ToString("x2")); + + return sha1Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/SHA256Context.cs b/RomRepoMgr.Core/Checksums/SHA256Context.cs new file mode 100644 index 0000000..bc7d232 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/SHA256Context.cs @@ -0,0 +1,132 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : SHA256Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Wraps up .NET SHA256 implementation to a Init(), Update(), Final() context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Wraps up .NET SHA256 implementation to a Init(), Update(), Final() context. + public sealed class Sha256Context : IChecksum + { + readonly SHA256 _provider; + + /// Initializes the SHA256 hash provider + public Sha256Context() => _provider = SHA256.Create(); + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) => _provider.TransformBlock(data, 0, (int)len, data, 0); + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + + return _provider.Hash; + } + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + var sha256Output = new StringBuilder(); + + foreach(byte h in _provider.Hash) + sha256Output.Append(h.ToString("x2")); + + return sha256Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + var localSha256Provider = SHA256.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + byte[] result = localSha256Provider.ComputeHash(fileStream); + fileStream.Close(); + + return result; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) + { + var localSha256Provider = SHA256.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + hash = localSha256Provider.ComputeHash(fileStream); + var sha256Output = new StringBuilder(); + + foreach(byte h in hash) + sha256Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return sha256Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) + { + var localSha256Provider = SHA256.Create(); + hash = localSha256Provider.ComputeHash(data, 0, (int)len); + var sha256Output = new StringBuilder(); + + foreach(byte h in hash) + sha256Output.Append(h.ToString("x2")); + + return sha256Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/SHA384Context.cs b/RomRepoMgr.Core/Checksums/SHA384Context.cs new file mode 100644 index 0000000..945720e --- /dev/null +++ b/RomRepoMgr.Core/Checksums/SHA384Context.cs @@ -0,0 +1,132 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : SHA384Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Wraps up .NET SHA384 implementation to a Init(), Update(), Final() context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Wraps up .NET SHA384 implementation to a Init(), Update(), Final() context. + public sealed class Sha384Context : IChecksum + { + readonly SHA384 _provider; + + /// Initializes the SHA384 hash provider + public Sha384Context() => _provider = SHA384.Create(); + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) => _provider.TransformBlock(data, 0, (int)len, data, 0); + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + + return _provider.Hash; + } + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + var sha384Output = new StringBuilder(); + + foreach(byte h in _provider.Hash) + sha384Output.Append(h.ToString("x2")); + + return sha384Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + var localSha384Provider = SHA384.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + byte[] result = localSha384Provider.ComputeHash(fileStream); + fileStream.Close(); + + return result; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) + { + var localSha384Provider = SHA384.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + hash = localSha384Provider.ComputeHash(fileStream); + var sha384Output = new StringBuilder(); + + foreach(byte h in hash) + sha384Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return sha384Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) + { + var localSha384Provider = SHA384.Create(); + hash = localSha384Provider.ComputeHash(data, 0, (int)len); + var sha384Output = new StringBuilder(); + + foreach(byte h in hash) + sha384Output.Append(h.ToString("x2")); + + return sha384Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/SHA512Context.cs b/RomRepoMgr.Core/Checksums/SHA512Context.cs new file mode 100644 index 0000000..d0040d7 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/SHA512Context.cs @@ -0,0 +1,132 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : SHA512Context.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Wraps up .NET SHA512 implementation to a Init(), Update(), Final() context. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Wraps up .NET SHA512 implementation to a Init(), Update(), Final() context. + public sealed class Sha512Context : IChecksum + { + readonly SHA512 _provider; + + /// Initializes the SHA512 hash provider + public Sha512Context() => _provider = SHA512.Create(); + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) => _provider.TransformBlock(data, 0, (int)len, data, 0); + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + + return _provider.Hash; + } + + /// + /// Returns a hexadecimal representation of the hash value. + public string End() + { + _provider.TransformFinalBlock(new byte[0], 0, 0); + var sha512Output = new StringBuilder(); + + foreach(byte h in _provider.Hash) + sha512Output.Append(h.ToString("x2")); + + return sha512Output.ToString(); + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) + { + var localSha512Provider = SHA512.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + byte[] result = localSha512Provider.ComputeHash(fileStream); + fileStream.Close(); + + return result; + } + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) + { + var localSha512Provider = SHA512.Create(); + var fileStream = new FileStream(filename, FileMode.Open); + hash = localSha512Provider.ComputeHash(fileStream); + var sha512Output = new StringBuilder(); + + foreach(byte h in hash) + sha512Output.Append(h.ToString("x2")); + + fileStream.Close(); + + return sha512Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// Byte array of the hash value. + public static string Data(byte[] data, uint len, out byte[] hash) + { + var localSha512Provider = SHA512.Create(); + hash = localSha512Provider.ComputeHash(data, 0, (int)len); + var sha512Output = new StringBuilder(); + + foreach(byte h in hash) + sha512Output.Append(h.ToString("x2")); + + return sha512Output.ToString(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Byte array of the hash value. + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Checksums/SpamSumContext.cs b/RomRepoMgr.Core/Checksums/SpamSumContext.cs new file mode 100644 index 0000000..5e1c769 --- /dev/null +++ b/RomRepoMgr.Core/Checksums/SpamSumContext.cs @@ -0,0 +1,526 @@ +// /*************************************************************************** +// Aaru Data Preservation Suite +// ---------------------------------------------------------------------------- +// +// Filename : SpamSumContext.cs +// Author(s) : Natalia Portillo +// +// Component : Checksums. +// +// --[ Description ] ---------------------------------------------------------- +// +// Implements the SpamSum fuzzy hashing algorithm. +// +// --[ License ] -------------------------------------------------------------- +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2011-2020 Natalia Portillo +// ****************************************************************************/ + +// Based on ssdeep +// Copyright (C) 2002 Andrew Tridgell +// Copyright (C) 2006 ManTech International Corporation +// Copyright (C) 2013 Helmut Grohne +// +// Earlier versions of this code were named fuzzy.c and can be found at: +// http://www.samba.org/ftp/unpacked/junkcode/spamsum/ +// http://ssdeep.sf.net/ + +using System; +using System.Runtime.CompilerServices; +using System.Text; +using Aaru.CommonTypes.Interfaces; + +namespace Aaru.Checksums +{ + /// Implements the SpamSum fuzzy hashing algorithm. + public sealed class SpamSumContext : IChecksum + { + const uint ROLLING_WINDOW = 7; + const uint MIN_BLOCKSIZE = 3; + const uint HASH_PRIME = 0x01000193; + const uint HASH_INIT = 0x28021967; + const uint NUM_BLOCKHASHES = 31; + const uint SPAMSUM_LENGTH = 64; + const uint FUZZY_MAX_RESULT = (2 * SPAMSUM_LENGTH) + 20; + + //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + readonly byte[] _b64 = + { + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, + 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, + 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x30, 0x31, + 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2F + }; + + FuzzyState _self; + + /// Initializes the SpamSum structures + public SpamSumContext() + { + _self = new FuzzyState + { + Bh = new BlockhashContext[NUM_BLOCKHASHES] + }; + + for(int i = 0; i < NUM_BLOCKHASHES; i++) + _self.Bh[i].Digest = new byte[SPAMSUM_LENGTH]; + + _self.Bhstart = 0; + _self.Bhend = 1; + _self.Bh[0].H = HASH_INIT; + _self.Bh[0].Halfh = HASH_INIT; + _self.Bh[0].Digest[0] = 0; + _self.Bh[0].Halfdigest = 0; + _self.Bh[0].Dlen = 0; + _self.TotalSize = 0; + roll_init(); + } + + /// + /// Updates the hash with data. + /// Data buffer. + /// Length of buffer to hash. + public void Update(byte[] data, uint len) + { + _self.TotalSize += len; + + for(int i = 0; i < len; i++) + fuzzy_engine_step(data[i]); + } + + /// + /// Updates the hash with data. + /// Data buffer. + public void Update(byte[] data) => Update(data, (uint)data.Length); + + /// + /// Returns a byte array of the hash value. + public byte[] Final() => throw new NotImplementedException("SpamSum does not have a binary representation."); + + /// + /// Returns a base64 representation of the hash value. + public string End() + { + FuzzyDigest(out byte[] result); + + return CToString(result); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void roll_init() => _self.Roll = new RollState + { + Window = new byte[ROLLING_WINDOW] + }; + + /* + * a rolling hash, based on the Adler checksum. By using a rolling hash + * we can perform auto resynchronisation after inserts/deletes + + * internally, h1 is the sum of the bytes in the window and h2 + * is the sum of the bytes times the index + + * h3 is a shift/xor based rolling hash, and is mostly needed to ensure that + * we can cope with large blocksize values + */ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void roll_hash(byte c) + { + _self.Roll.H2 -= _self.Roll.H1; + _self.Roll.H2 += ROLLING_WINDOW * c; + + _self.Roll.H1 += c; + _self.Roll.H1 -= _self.Roll.Window[_self.Roll.N % ROLLING_WINDOW]; + + _self.Roll.Window[_self.Roll.N % ROLLING_WINDOW] = c; + _self.Roll.N++; + + /* The original spamsum AND'ed this value with 0xFFFFFFFF which + * in theory should have no effect. This AND has been removed + * for performance (jk) */ + _self.Roll.H3 <<= 5; + _self.Roll.H3 ^= c; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + uint roll_sum() => _self.Roll.H1 + _self.Roll.H2 + _self.Roll.H3; + + /* A simple non-rolling hash, based on the FNV hash. */ + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static uint sum_hash(byte c, uint h) => (h * HASH_PRIME) ^ c; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static uint SSDEEP_BS(uint index) => MIN_BLOCKSIZE << (int)index; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void fuzzy_try_fork_blockhash() + { + if(_self.Bhend >= NUM_BLOCKHASHES) + return; + + if(_self.Bhend == 0) // assert + throw new Exception("Assertion failed"); + + uint obh = _self.Bhend - 1; + uint nbh = _self.Bhend; + _self.Bh[nbh].H = _self.Bh[obh].H; + _self.Bh[nbh].Halfh = _self.Bh[obh].Halfh; + _self.Bh[nbh].Digest[0] = 0; + _self.Bh[nbh].Halfdigest = 0; + _self.Bh[nbh].Dlen = 0; + ++_self.Bhend; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void fuzzy_try_reduce_blockhash() + { + if(_self.Bhstart >= _self.Bhend) + throw new Exception("Assertion failed"); + + if(_self.Bhend - _self.Bhstart < 2) + /* Need at least two working hashes. */ + return; + + if((ulong)SSDEEP_BS(_self.Bhstart) * SPAMSUM_LENGTH >= _self.TotalSize) + /* Initial blocksize estimate would select this or a smaller + * blocksize. */ + return; + + if(_self.Bh[_self.Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2) + /* Estimate adjustment would select this blocksize. */ + return; + + /* At this point we are clearly no longer interested in the + * start_blocksize. Get rid of it. */ + ++_self.Bhstart; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void fuzzy_engine_step(byte c) + { + uint i; + /* At each character we update the rolling hash and the normal hashes. + * When the rolling hash hits a reset value then we emit a normal hash + * as a element of the signature and reset the normal hash. */ + roll_hash(c); + ulong h = roll_sum(); + + for(i = _self.Bhstart; i < _self.Bhend; ++i) + { + _self.Bh[i].H = sum_hash(c, _self.Bh[i].H); + _self.Bh[i].Halfh = sum_hash(c, _self.Bh[i].Halfh); + } + + for(i = _self.Bhstart; i < _self.Bhend; ++i) + { + /* With growing blocksize almost no runs fail the next test. */ + if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1) + /* Once this condition is false for one bs, it is + * automatically false for all further bs. I.e. if + * h === -1 (mod 2*bs) then h === -1 (mod bs). */ + break; + + /* We have hit a reset point. We now emit hashes which are + * based on all characters in the piece of the message between + * the last reset point and this one */ + if(0 == _self.Bh[i].Dlen) + fuzzy_try_fork_blockhash(); + + _self.Bh[i].Digest[_self.Bh[i].Dlen] = _b64[_self.Bh[i].H % 64]; + _self.Bh[i].Halfdigest = _b64[_self.Bh[i].Halfh % 64]; + + if(_self.Bh[i].Dlen < SPAMSUM_LENGTH - 1) + { + /* We can have a problem with the tail overflowing. The + * easiest way to cope with this is to only reset the + * normal hash if we have room for more characters in + * our signature. This has the effect of combining the + * last few pieces of the message into a single piece + * */ + _self.Bh[i].Digest[++_self.Bh[i].Dlen] = 0; + _self.Bh[i].H = HASH_INIT; + + if(_self.Bh[i].Dlen >= SPAMSUM_LENGTH / 2) + continue; + + _self.Bh[i].Halfh = HASH_INIT; + _self.Bh[i].Halfdigest = 0; + } + else + fuzzy_try_reduce_blockhash(); + } + } + + // CLAUNIA: Flags seems to never be used in ssdeep, so I just removed it for code simplicity + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void FuzzyDigest(out byte[] result) + { + var sb = new StringBuilder(); + uint bi = _self.Bhstart; + uint h = roll_sum(); + int remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */ + result = new byte[FUZZY_MAX_RESULT]; + + /* Verify that our elimination was not overeager. */ + if(!(bi == 0 || ((ulong)SSDEEP_BS(bi) / 2) * SPAMSUM_LENGTH < _self.TotalSize)) + throw new Exception("Assertion failed"); + + int resultOff = 0; + + /* Initial blocksize guess. */ + while((ulong)SSDEEP_BS(bi) * SPAMSUM_LENGTH < _self.TotalSize) + { + ++bi; + + if(bi >= NUM_BLOCKHASHES) + throw new OverflowException("The input exceeds data types."); + } + + /* Adapt blocksize guess to actual digest length. */ + while(bi >= _self.Bhend) + --bi; + + while(bi > _self.Bhstart && + _self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2) + --bi; + + if(bi > 0 && + _self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2) + throw new Exception("Assertion failed"); + + sb.AppendFormat("{0}:", SSDEEP_BS(bi)); + int i = Encoding.ASCII.GetBytes(sb.ToString()).Length; + + if(i <= 0) + /* Maybe snprintf has set errno here? */ + throw new OverflowException("The input exceeds data types."); + + if(i >= remain) + throw new Exception("Assertion failed"); + + remain -= i; + + Array.Copy(Encoding.ASCII.GetBytes(sb.ToString()), 0, result, 0, i); + + resultOff = i; + + i = (int)_self.Bh[bi].Dlen; + + if(i > remain) + throw new Exception("Assertion failed"); + + Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i); + resultOff += i; + remain -= i; + + if(h != 0) + { + if(remain <= 0) + throw new Exception("Assertion failed"); + + result[resultOff] = _b64[_self.Bh[bi].H % 64]; + + if(i < 3 || + result[resultOff] != result[resultOff - 1] || + result[resultOff] != result[resultOff - 2] || + result[resultOff] != result[resultOff - 3]) + { + ++resultOff; + --remain; + } + } + else if(_self.Bh[bi].Digest[i] != 0) + { + if(remain <= 0) + throw new Exception("Assertion failed"); + + result[resultOff] = _self.Bh[bi].Digest[i]; + + if(i < 3 || + result[resultOff] != result[resultOff - 1] || + result[resultOff] != result[resultOff - 2] || + result[resultOff] != result[resultOff - 3]) + { + ++resultOff; + --remain; + } + } + + if(remain <= 0) + throw new Exception("Assertion failed"); + + result[resultOff++] = 0x3A; // ':' + --remain; + + if(bi < _self.Bhend - 1) + { + ++bi; + i = (int)_self.Bh[bi].Dlen; + + if(i > remain) + throw new Exception("Assertion failed"); + + Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i); + resultOff += i; + remain -= i; + + if(h != 0) + { + if(remain <= 0) + throw new Exception("Assertion failed"); + + h = _self.Bh[bi].Halfh; + result[resultOff] = _b64[h % 64]; + + if(i < 3 || + result[resultOff] != result[resultOff - 1] || + result[resultOff] != result[resultOff - 2] || + result[resultOff] != result[resultOff - 3]) + { + ++resultOff; + --remain; + } + } + else + { + i = _self.Bh[bi].Halfdigest; + + if(i != 0) + { + if(remain <= 0) + throw new Exception("Assertion failed"); + + result[resultOff] = (byte)i; + + if(i < 3 || + result[resultOff] != result[resultOff - 1] || + result[resultOff] != result[resultOff - 2] || + result[resultOff] != result[resultOff - 3]) + { + ++resultOff; + --remain; + } + } + } + } + else if(h != 0) + { + if(_self.Bh[bi].Dlen != 0) + throw new Exception("Assertion failed"); + + if(remain <= 0) + throw new Exception("Assertion failed"); + + result[resultOff++] = _b64[_self.Bh[bi].H % 64]; + /* No need to bother with FUZZY_FLAG_ELIMSEQ, because this + * digest has length 1. */ + --remain; + } + + result[resultOff] = 0; + } + + /// Gets the hash of a file + /// File path. + public static byte[] File(string filename) => + throw new NotImplementedException("SpamSum does not have a binary representation."); + + /// Gets the hash of a file in hexadecimal and as a byte array. + /// File path. + /// Byte array of the hash value. + public static string File(string filename, out byte[] hash) => + throw new NotImplementedException("Not yet implemented."); + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// Length of the data buffer to hash. + /// null + /// Base64 representation of SpamSum $blocksize:$hash:$hash + public static string Data(byte[] data, uint len, out byte[] hash) + { + var fuzzyContext = new SpamSumContext(); + + fuzzyContext.Update(data, len); + + hash = null; + + return fuzzyContext.End(); + } + + /// Gets the hash of the specified data buffer. + /// Data buffer. + /// null + /// Base64 representation of SpamSum $blocksize:$hash:$hash + public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash); + + // Converts an ASCII null-terminated string to .NET string + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static string CToString(byte[] cString) + { + int count = 0; + + // ReSharper disable once LoopCanBeConvertedToQuery + // LINQ is six times slower + foreach(byte c in cString) + { + if(c == 0) + break; + + count++; + } + + return Encoding.ASCII.GetString(cString, 0, count); + } + + struct RollState + { + public byte[] Window; + + // ROLLING_WINDOW + public uint H1; + public uint H2; + public uint H3; + public uint N; + } + + /* A blockhash contains a signature state for a specific (implicit) blocksize. + * The blocksize is given by SSDEEP_BS(index). The h and halfh members are the + * FNV hashes, where halfh stops to be reset after digest is SPAMSUM_LENGTH/2 + * long. The halfh hash is needed be able to truncate digest for the second + * output hash to stay compatible with ssdeep output. */ + struct BlockhashContext + { + public uint H; + public uint Halfh; + public byte[] Digest; + + // SPAMSUM_LENGTH + public byte Halfdigest; + public uint Dlen; + } + + struct FuzzyState + { + public uint Bhstart; + public uint Bhend; + public BlockhashContext[] Bh; + + //NUM_BLOCKHASHES + public ulong TotalSize; + public RollState Roll; + } + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/RomRepoMgr.Core.csproj b/RomRepoMgr.Core/RomRepoMgr.Core.csproj index 44f44d0..2914fc9 100644 --- a/RomRepoMgr.Core/RomRepoMgr.Core.csproj +++ b/RomRepoMgr.Core/RomRepoMgr.Core.csproj @@ -5,10 +5,13 @@ - + + + + diff --git a/RomRepoMgr.Core/Workers/Compression.cs b/RomRepoMgr.Core/Workers/Compression.cs new file mode 100644 index 0000000..c660bb9 --- /dev/null +++ b/RomRepoMgr.Core/Workers/Compression.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using RomRepoMgr.Core.EventArgs; +using SharpCompress.Compressors; +using SharpCompress.Compressors.LZMA; + +namespace RomRepoMgr.Core.Workers +{ + public sealed class Compression + { + const long BUFFER_SIZE = 131072; + + public event EventHandler SetProgressBounds; + public event EventHandler SetProgress; + + public void CompressFile(string source, string destination) + { + var inFs = new FileStream(source, FileMode.Open, FileAccess.Read); + var outFs = new FileStream(destination, FileMode.CreateNew, FileAccess.Write); + Stream zStream = new LZipStream(outFs, CompressionMode.Compress); + + byte[] buffer = new byte[BUFFER_SIZE]; + + SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = inFs.Length + }); + + while(inFs.Position + BUFFER_SIZE <= inFs.Length) + { + SetProgress?.Invoke(this, new ProgressEventArgs + { + Value = inFs.Position + }); + + inFs.Read(buffer, 0, buffer.Length); + zStream.Write(buffer, 0, buffer.Length); + } + + buffer = new byte[inFs.Length - inFs.Position]; + + SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = inFs.Length + }); + + inFs.Read(buffer, 0, buffer.Length); + zStream.Write(buffer, 0, buffer.Length); + + inFs.Close(); + zStream.Close(); + outFs.Dispose(); + } + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Workers/DatImporter.cs b/RomRepoMgr.Core/Workers/DatImporter.cs index 9312087..b6355d5 100644 --- a/RomRepoMgr.Core/Workers/DatImporter.cs +++ b/RomRepoMgr.Core/Workers/DatImporter.cs @@ -25,17 +25,25 @@ using System; using System.Diagnostics; +using System.IO; +using Aaru.Checksums; using RomRepoMgr.Core.EventArgs; using SabreTools.Library.DatFiles; +using ErrorEventArgs = RomRepoMgr.Core.EventArgs.ErrorEventArgs; namespace RomRepoMgr.Core.Workers { public sealed class DatImporter { + readonly string _datFilesPath; readonly string _datPath; bool _aborted; - public DatImporter(string datPath) => _datPath = datPath; + public DatImporter(string datPath) + { + _datPath = datPath; + _datFilesPath = Path.Combine(Settings.Settings.Current.RepositoryPath, "datfiles"); + } public void Import() { @@ -53,6 +61,38 @@ namespace RomRepoMgr.Core.Workers DateTime end = DateTime.UtcNow; double elapsed = (end - start).TotalSeconds; + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Hashing DAT file..." + }); + + string datHash = Sha384Context.File(_datPath, out _); + + if(!Directory.Exists(_datFilesPath)) + Directory.CreateDirectory(_datFilesPath); + + string compressedDatPath = Path.Combine(_datFilesPath, datHash + ".lz"); + + if(File.Exists(compressedDatPath)) + { + ErrorOccurred?.Invoke(this, new ErrorEventArgs + { + Message = "DAT file is already in database, not importing duplicates." + }); + + return; + } + + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Compressing DAT file..." + }); + + var datCompress = new Compression(); + datCompress.SetProgress += SetProgress; + datCompress.SetProgressBounds += SetProgressBounds; + datCompress.CompressFile(_datPath, compressedDatPath); + WorkFinished?.Invoke(this, System.EventArgs.Empty); } catch(Exception e) diff --git a/RomRepoMgr.Core/Interop/DetectOS.cs b/RomRepoMgr.Settings/Interop/DetectOS.cs similarity index 100% rename from RomRepoMgr.Core/Interop/DetectOS.cs rename to RomRepoMgr.Settings/Interop/DetectOS.cs diff --git a/RomRepoMgr.Core/Interop/PlatformID.cs b/RomRepoMgr.Settings/Interop/PlatformID.cs similarity index 100% rename from RomRepoMgr.Core/Interop/PlatformID.cs rename to RomRepoMgr.Settings/Interop/PlatformID.cs diff --git a/RomRepoMgr.Core/Interop/Version.cs b/RomRepoMgr.Settings/Interop/Version.cs similarity index 100% rename from RomRepoMgr.Core/Interop/Version.cs rename to RomRepoMgr.Settings/Interop/Version.cs diff --git a/RomRepoMgr.Settings/RomRepoMgr.Settings.csproj b/RomRepoMgr.Settings/RomRepoMgr.Settings.csproj index 55095ea..3d40b44 100644 --- a/RomRepoMgr.Settings/RomRepoMgr.Settings.csproj +++ b/RomRepoMgr.Settings/RomRepoMgr.Settings.csproj @@ -4,10 +4,6 @@ netcoreapp3.1 - - - -