claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add and use SpamSum for Media and Rom types

Browse Source 
SpamSum implementation courtesy of https://github.com/aaru-dps/Aaru
This commit is contained in:
Matt Nadareski
2020-09-04 15:02:15 -07:00
parent 40ca4e39a5
commit 5f749d07a4
26 changed files with 834 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
SabreTools.Library/DatFiles/ClrMamePro.cs
View File

@@ -340,6 +340,13 @@ namespace SabreTools.Library.DatFiles
if (item.ItemType == ItemType.Rom)
(item as Rom).SHA512 = attrVal;
break;
case "spamsum":
if (item.ItemType == ItemType.Media)
(item as Media).SpamSum = attrVal;
else if (item.ItemType == ItemType.Rom)
(item as Rom).SpamSum = attrVal;
break;
case "status":
ItemStatus tempFlagStatus = attrVal.AsItemStatus();
@@ -634,6 +641,7 @@ namespace SabreTools.Library.DatFiles
cmpw.WriteOptionalAttributeString("md5", media.MD5?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("sha1", media.SHA1?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("sha256", media.SHA256?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("spamsum", media.SpamSum?.ToLowerInvariant());
cmpw.WriteEndElement();
break;
@@ -662,6 +670,7 @@ namespace SabreTools.Library.DatFiles
cmpw.WriteOptionalAttributeString("sha256", rom.SHA256?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("sha384", rom.SHA384?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("sha512", rom.SHA512?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("spamsum", rom.SpamSum?.ToLowerInvariant());
cmpw.WriteOptionalAttributeString("date", rom.Date);
cmpw.WriteOptionalAttributeString("flags", rom.ItemStatus.FromItemStatus(false));
cmpw.WriteEndElement();

11
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -129,6 +129,9 @@ namespace SabreTools.Library.DatFiles
case DatFormat.RedumpSHA512:
return new Hashfile(baseDat, Hash.SHA512);
case DatFormat.RedumpSpamSum:
return new Hashfile(baseDat, Hash.SpamSum);
case DatFormat.RomCenter:
return new RomCenter(baseDat);
@@ -3491,7 +3494,8 @@ namespace SabreTools.Library.DatFiles
sha256 = string.Empty,
sha384 = string.Empty,
sha512 = string.Empty,
size = string.Empty;
size = string.Empty,
spamsum = string.Empty;
// If we have a prefix
if (prefix)
@@ -3512,6 +3516,7 @@ namespace SabreTools.Library.DatFiles
md5 = (item as Media).MD5 ?? string.Empty;
sha1 = (item as Media).SHA1 ?? string.Empty;
sha256 = (item as Media).SHA256 ?? string.Empty;
spamsum = (item as Media).SpamSum ?? string.Empty;
}
else if (item.ItemType == ItemType.Rom)
{
@@ -3525,6 +3530,7 @@ namespace SabreTools.Library.DatFiles
sha384 = (item as Rom).SHA384 ?? string.Empty;
sha512 = (item as Rom).SHA512 ?? string.Empty;
size = (item as Rom).Size.ToString();
spamsum = (item as Rom).SpamSum ?? string.Empty;
}
// Now do bulk replacement where possible
@@ -3542,7 +3548,8 @@ namespace SabreTools.Library.DatFiles
.Replace("%sha256%", sha256)
.Replace("%sha384%", sha384)
.Replace("%sha512%", sha512)
.Replace("%size%", size);
.Replace("%size%", size)
.Replace("%spamsum%", spamsum);
// TODO: Add GameName logic here too?
// TODO: Figure out what I meant by the above ^

11
SabreTools.Library/DatFiles/DatHeader.cs
View File

@@ -796,6 +796,17 @@ namespace SabreTools.Library.DatFiles
#endregion
#region .spamsum
// Redump SpamSum
if (DatFormat.HasFlag(DatFormat.RedumpSpamSum))
{
outfileNames.Add(DatFormat.RedumpSpamSum, CreateOutFileNamesHelper(outDir, ".spamsum", overwrite));
usedExtensions.Add(".spamsum");
};
#endregion
#region .ssv
// SSV

14
SabreTools.Library/DatFiles/Enums.cs
View File

@@ -149,6 +149,11 @@ namespace SabreTools.Library.DatFiles
/// </summary>
RedumpSHA512 = 1 << 24,
/// <summary>
/// SpamSum hash list
/// </summary>
RedumpSpamSum = 1 << 25,
#endregion
// Specialty combinations
@@ -191,15 +196,16 @@ namespace SabreTools.Library.DatFiles
SHA256 = 1 << 4,
SHA384 = 1 << 5,
SHA512 = 1 << 6,
SpamSum = 1 << 7,
// Special combinations
Standard = CRC | MD5 | SHA1,
#if NET_FRAMEWORK
DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512,
SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512,
DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum,
SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
#else
DeepHashes = SHA256 | SHA384 | SHA512,
SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512,
DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum,
SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum,
#endif
}

19
SabreTools.Library/DatFiles/Hashfile.cs
View File

@@ -75,6 +75,7 @@ namespace SabreTools.Library.DatFiles
SHA256 = (_hash.HasFlag(Hash.SHA256) ? hash : null),
SHA384 = (_hash.HasFlag(Hash.SHA384) ? hash : null),
SHA512 = (_hash.HasFlag(Hash.SHA512) ? hash : null),
SpamSum = (_hash.HasFlag(Hash.SpamSum) ? hash : null),
ItemStatus = ItemStatus.None,
Machine = new Machine
@@ -316,6 +317,24 @@ namespace SabreTools.Library.DatFiles
break;
}
break;
case Hash.SpamSum:
switch (datItem.ItemType)
{
case ItemType.Media:
var media = datItem as Media;
fields[0] = media.SpamSum;
fields[1] = name;
break;
case ItemType.Rom:
var rom = datItem as Rom;
fields[0] = rom.SpamSum;
fields[1] = name;
break;
}
break;
}

8
SabreTools.Library/DatFiles/ItemDictionary.cs
View File

@@ -314,6 +314,12 @@ namespace SabreTools.Library.DatFiles
[JsonIgnore]
public long SHA512Count { get; private set; } = 0;
/// <summary>
/// Number of items with a SpamSum fuzzy hash
/// </summary>
[JsonIgnore]
public long SpamSumCount { get; private set; } = 0;
/// <summary>
/// Number of items with the baddump status
/// </summary>
@@ -652,6 +658,7 @@ namespace SabreTools.Library.DatFiles
MD5Count += (string.IsNullOrWhiteSpace((item as Media).MD5) ? 0 : 1);
SHA1Count += (string.IsNullOrWhiteSpace((item as Media).SHA1) ? 0 : 1);
SHA256Count += (string.IsNullOrWhiteSpace((item as Media).SHA256) ? 0 : 1);
SpamSumCount += (string.IsNullOrWhiteSpace((item as Media).SpamSum) ? 0 : 1);
break;
case ItemType.Part:
PartCount++;
@@ -682,6 +689,7 @@ namespace SabreTools.Library.DatFiles
SHA256Count += (string.IsNullOrWhiteSpace((item as Rom).SHA256) ? 0 : 1);
SHA384Count += (string.IsNullOrWhiteSpace((item as Rom).SHA384) ? 0 : 1);
SHA512Count += (string.IsNullOrWhiteSpace((item as Rom).SHA512) ? 0 : 1);
SpamSumCount += (string.IsNullOrWhiteSpace((item as Rom).SpamSum) ? 0 : 1);
}
BaddumpCount += ((item as Rom).ItemStatus == ItemStatus.BadDump ? 1 : 0);

4
SabreTools.Library/DatFiles/Logiqx.cs
View File

@@ -446,6 +446,7 @@ namespace SabreTools.Library.DatFiles
MD5 = reader.GetAttribute("md5"),
SHA1 = reader.GetAttribute("sha1"),
SHA256 = reader.GetAttribute("sha256"),
SpamSum = reader.GetAttribute("spamsum"),
Source = new Source
{
@@ -498,6 +499,7 @@ namespace SabreTools.Library.DatFiles
SHA256 = reader.GetAttribute("sha256"),
SHA384 = reader.GetAttribute("sha384"),
SHA512 = reader.GetAttribute("sha512"),
SpamSum = reader.GetAttribute("spamsum"),
MergeTag = reader.GetAttribute("merge"),
ItemStatus = reader.GetAttribute("status").AsItemStatus(),
Date = Sanitizer.CleanDate(reader.GetAttribute("date")),
@@ -970,6 +972,7 @@ namespace SabreTools.Library.DatFiles
xtw.WriteOptionalAttributeString("md5", media.MD5?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha1", media.SHA1?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha256", media.SHA256?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("spamsum", media.SpamSum?.ToLowerInvariant());
xtw.WriteEndElement();
break;
@@ -998,6 +1001,7 @@ namespace SabreTools.Library.DatFiles
xtw.WriteOptionalAttributeString("sha256", rom.SHA256?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha384", rom.SHA384?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha512", rom.SHA512?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("spamsum", rom.SpamSum?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("date", rom.Date);
xtw.WriteOptionalAttributeString("status", rom.ItemStatus.FromItemStatus(false));
xtw.WriteOptionalAttributeString("inverted", rom.Inverted.FromYesNo());

4
SabreTools.Library/DatFiles/SabreDat.cs
View File

@@ -485,6 +485,7 @@ namespace SabreTools.Library.DatFiles
MD5 = reader.GetAttribute("md5"),
SHA1 = reader.GetAttribute("sha1"),
SHA256 = reader.GetAttribute("sha256"),
SpamSum = reader.GetAttribute("spamsum"),
Source = new Source
{
@@ -525,6 +526,7 @@ namespace SabreTools.Library.DatFiles
SHA256 = reader.GetAttribute("sha256"),
SHA384 = reader.GetAttribute("sha384"),
SHA512 = reader.GetAttribute("sha512"),
SpamSum = reader.GetAttribute("spamsum"),
ItemStatus = its,
Date = date,
@@ -1528,6 +1530,7 @@ namespace SabreTools.Library.DatFiles
xtw.WriteOptionalAttributeString("md5", media.MD5?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha1", media.SHA1?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha256", media.SHA256?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("spamsum", media.SpamSum?.ToLowerInvariant());
xtw.WriteEndElement();
break;
@@ -1585,6 +1588,7 @@ namespace SabreTools.Library.DatFiles
xtw.WriteOptionalAttributeString("sha256", rom.SHA256?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha384", rom.SHA384?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("sha512", rom.SHA512?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("spamsum", rom.SpamSum?.ToLowerInvariant());
xtw.WriteOptionalAttributeString("date", rom.Date);
if (rom.ItemStatus != ItemStatus.None)
{

6
SabreTools.Library/DatFiles/SeparatedValue.cs
View File

@@ -187,6 +187,7 @@ namespace SabreTools.Library.DatFiles
"SHA256",
//"SHA384",
//"SHA512",
//"SpamSum",
"Nodump",
};
@@ -219,7 +220,7 @@ namespace SabreTools.Library.DatFiles
// Build the state
// TODO: Can we have some way of saying what fields to write out? Support for read extends to all fields now
string[] fields = new string[14]; // 17;
string[] fields = new string[14]; // 18;
fields[0] = Header.FileName;
fields[1] = Header.Name;
fields[2] = Header.Description;
@@ -241,6 +242,7 @@ namespace SabreTools.Library.DatFiles
fields[12] = string.Empty;
//fields[13] = string.Empty;
//fields[14] = string.Empty;
//fields[15] = string.Empty;
fields[13] = disk.ItemStatus.ToString();
break;
@@ -257,6 +259,7 @@ namespace SabreTools.Library.DatFiles
fields[12] = media.SHA256?.ToLowerInvariant();
//fields[13] = string.Empty;
//fields[14] = string.Empty;
//fields[15] = media.SpamSum?.ToLowerInvariant();
fields[13] = string.Empty;
break;
@@ -273,6 +276,7 @@ namespace SabreTools.Library.DatFiles
fields[12] = rom.SHA256?.ToLowerInvariant();
//fields[13] = rom.SHA384?.ToLowerInvariant();
//fields[14] = rom.SHA512?.ToLowerInvariant();
//fields[15] = rom.SpamSum?.ToLowerInvariant();
fields[13] = rom.ItemStatus.ToString();
break;
}

5
SabreTools.Library/DatItems/DatItem.cs
View File

@@ -94,6 +94,7 @@ namespace SabreTools.Library.DatItems
Field.DatItem_SHA256,
Field.DatItem_SHA384,
Field.DatItem_SHA512,
Field.DatItem_SpamSum,
Field.DatItem_Merge,
Field.DatItem_Region,
Field.DatItem_Offset,
@@ -782,6 +783,10 @@ namespace SabreTools.Library.DatItems
case Field.DatItem_SHA512:
key = Constants.SHA512Zero;
break;
case Field.DatItem_SpamSum:
key = null;
break;
}
// Double and triple check the key for corner cases

1
SabreTools.Library/DatItems/Enums.cs
View File

@@ -282,6 +282,7 @@ namespace SabreTools.Library.DatItems
DatItem_SHA256,
DatItem_SHA384,
DatItem_SHA512,
DatItem_SpamSum,
DatItem_Merge,
DatItem_Region,
DatItem_Offset,

53
SabreTools.Library/DatItems/Media.cs
View File

@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using SabreTools.Library.FileTypes;
using SabreTools.Library.Filtering;
@@ -20,7 +21,7 @@ namespace SabreTools.Library.DatItems
private byte[] _md5; // 16 bytes
private byte[] _sha1; // 20 bytes
private byte[] _sha256; // 32 bytes
// TODO: Implement SpamSum
private byte[] _spamsum; // variable bytes
#endregion
@@ -62,6 +63,16 @@ namespace SabreTools.Library.DatItems
set { _sha256 = Utilities.StringToByteArray(Sanitizer.CleanSHA256(value)); }
}
/// <summary>
/// File SpamSum fuzzy hash
/// </summary>
[JsonProperty("spamsum", DefaultValueHandling = DefaultValueHandling.Ignore)]
public string SpamSum
{
get { return _spamsum.IsNullOrEmpty() ? null : Encoding.UTF8.GetString(_spamsum); }
set { _spamsum = Encoding.UTF8.GetBytes(value); }
}
#endregion
#region Accessors
@@ -96,6 +107,9 @@ namespace SabreTools.Library.DatItems
if (mappings.Keys.Contains(Field.DatItem_SHA256))
SHA256 = mappings[Field.DatItem_SHA256];
if (mappings.Keys.Contains(Field.DatItem_SpamSum))
SpamSum = mappings[Field.DatItem_SpamSum];
}
#endregion
@@ -122,6 +136,7 @@ namespace SabreTools.Library.DatItems
_md5 = baseFile.MD5;
_sha1 = baseFile.SHA1;
_sha256 = baseFile.SHA256;
_spamsum = baseFile.SpamSum;
ItemType = ItemType.Media;
DupeType = 0x00;
@@ -146,11 +161,12 @@ namespace SabreTools.Library.DatItems
_md5 = this._md5,
_sha1 = this._sha1,
_sha256 = this._sha256,
_spamsum = this._spamsum,
};
}
/// <summary>
/// Convert a disk to the closest Rom approximation
/// Convert a media to the closest Rom approximation
/// </summary>
/// <returns></returns>
public Rom ConvertToRom()
@@ -168,6 +184,7 @@ namespace SabreTools.Library.DatItems
MD5 = this.MD5,
SHA1 = this.SHA1,
SHA256 = this.SHA256,
SpamSum = this.SpamSum,
};
return rom;
@@ -209,6 +226,9 @@ namespace SabreTools.Library.DatItems
if (_sha256.IsNullOrEmpty() && !other._sha256.IsNullOrEmpty())
_sha256 = other._sha256;
if (_spamsum.IsNullOrEmpty() && !other._spamsum.IsNullOrEmpty())
_spamsum = other._spamsum;
}
/// <summary>
@@ -223,6 +243,8 @@ namespace SabreTools.Library.DatItems
return $"_{SHA1}";
else if (!_sha256.IsNullOrEmpty())
return $"_{SHA256}";
else if (!_spamsum.IsNullOrEmpty())
return $"_{SpamSum}";
else
return "_1";
}
@@ -236,7 +258,8 @@ namespace SabreTools.Library.DatItems
{
return !(_md5.IsNullOrEmpty() ^ other._md5.IsNullOrEmpty())
|| !(_sha1.IsNullOrEmpty() ^ other._sha1.IsNullOrEmpty())
|| !(_sha256.IsNullOrEmpty() ^ other._sha256.IsNullOrEmpty());
|| !(_sha256.IsNullOrEmpty() ^ other._sha256.IsNullOrEmpty())
|| !(_spamsum.IsNullOrEmpty() ^ other._spamsum.IsNullOrEmpty());
}
/// <summary>
@@ -247,7 +270,8 @@ namespace SabreTools.Library.DatItems
{
return !_md5.IsNullOrEmpty()
|| !_sha1.IsNullOrEmpty()
|| !_sha256.IsNullOrEmpty();
|| !_sha256.IsNullOrEmpty()
|| !_spamsum.IsNullOrEmpty();
}
/// <summary>
@@ -268,7 +292,7 @@ namespace SabreTools.Library.DatItems
// Return if all hashes match according to merge rules
return ConditionalHashEquals(_md5, other._md5)
&& ConditionalHashEquals(_sha1, other._sha1)
&& ConditionalHashEquals(_sha256, other._sha256);
&& ConditionalHashEquals(_spamsum, other._spamsum);
}
#endregion
@@ -337,6 +361,12 @@ namespace SabreTools.Library.DatItems
if (filter.DatItem_SHA256.MatchesNegativeSet(SHA256) == true)
return false;
// Filter on SpamSum
if (filter.DatItem_SpamSum.MatchesPositiveSet(SpamSum) == false)
return false;
if (filter.DatItem_SpamSum.MatchesNegativeSet(SpamSum) == true)
return false;
return true;
}
@@ -361,6 +391,9 @@ namespace SabreTools.Library.DatItems
if (fields.Contains(Field.DatItem_SHA256))
SHA256 = null;
if (fields.Contains(Field.DatItem_SpamSum))
SpamSum = null;
}
/// <summary>
@@ -404,6 +437,10 @@ namespace SabreTools.Library.DatItems
key = SHA256;
break;
case Field.DatItem_SpamSum:
key = SpamSum;
break;
// Let the base handle generic stuff
default:
return base.GetKey(bucketedBy, lower, norename);
@@ -454,6 +491,12 @@ namespace SabreTools.Library.DatItems
if (string.IsNullOrEmpty(SHA256) && !string.IsNullOrEmpty(newItem.SHA256))
SHA256 = newItem.SHA256;
}
if (fields.Contains(Field.DatItem_SpamSum))
{
if (string.IsNullOrEmpty(SpamSum) && !string.IsNullOrEmpty(newItem.SpamSum))
SpamSum = newItem.SpamSum;
}
}
#endregion

50
SabreTools.Library/DatItems/Rom.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using SabreTools.Library.Data;
using SabreTools.Library.FileTypes;
@@ -29,6 +30,7 @@ namespace SabreTools.Library.DatItems
private byte[] _sha256; // 32 bytes
private byte[] _sha384; // 48 bytes
private byte[] _sha512; // 64 bytes
private byte[] _spamsum; // variable bytes
#endregion
@@ -127,6 +129,16 @@ namespace SabreTools.Library.DatItems
set { _sha512 = Utilities.StringToByteArray(Sanitizer.CleanSHA512(value)); }
}
/// <summary>
/// File SpamSum fuzzy hash
/// </summary>
[JsonProperty("spamsum", DefaultValueHandling = DefaultValueHandling.Ignore)]
public string SpamSum
{
get { return _spamsum.IsNullOrEmpty() ? null : Encoding.UTF8.GetString(_spamsum); }
set { _spamsum = Encoding.UTF8.GetBytes(value); }
}
/// <summary>
/// Rom name to merge from parent
/// </summary>
@@ -314,6 +326,9 @@ namespace SabreTools.Library.DatItems
if (mappings.Keys.Contains(Field.DatItem_SHA512))
SHA512 = mappings[Field.DatItem_SHA512];
if (mappings.Keys.Contains(Field.DatItem_SpamSum))
SpamSum = mappings[Field.DatItem_SpamSum];
if (mappings.Keys.Contains(Field.DatItem_Merge))
MergeTag = mappings[Field.DatItem_Merge];
@@ -441,6 +456,7 @@ namespace SabreTools.Library.DatItems
_sha256 = baseFile.SHA256;
_sha384 = baseFile.SHA384;
_sha512 = baseFile.SHA512;
_spamsum = baseFile.SpamSum;
ItemType = ItemType.Rom;
DupeType = 0x00;
@@ -475,6 +491,7 @@ namespace SabreTools.Library.DatItems
_sha256 = this._sha256,
_sha384 = this._sha384,
_sha512 = this._sha512,
_spamsum = this._spamsum,
MergeTag = this.MergeTag,
Region = this.Region,
Offset = this.Offset,
@@ -568,6 +585,9 @@ namespace SabreTools.Library.DatItems
if (_sha512.IsNullOrEmpty() && !other._sha512.IsNullOrEmpty())
_sha512 = other._sha512;
if (_spamsum.IsNullOrEmpty() && !other._spamsum.IsNullOrEmpty())
_spamsum = other._spamsum;
}
/// <summary>
@@ -588,6 +608,8 @@ namespace SabreTools.Library.DatItems
return $"_{SHA384}";
else if (!_sha512.IsNullOrEmpty())
return $"_{SHA512}";
else if (!_spamsum.IsNullOrEmpty())
return $"_{SpamSum}";
else
return "_1";
}
@@ -607,7 +629,8 @@ namespace SabreTools.Library.DatItems
|| !(_sha1.IsNullOrEmpty() ^ other._sha1.IsNullOrEmpty())
|| !(_sha256.IsNullOrEmpty() ^ other._sha256.IsNullOrEmpty())
|| !(_sha384.IsNullOrEmpty() ^ other._sha384.IsNullOrEmpty())
|| !(_sha512.IsNullOrEmpty() ^ other._sha512.IsNullOrEmpty());
|| !(_sha512.IsNullOrEmpty() ^ other._sha512.IsNullOrEmpty())
|| !(_spamsum.IsNullOrEmpty() ^ other._spamsum.IsNullOrEmpty());
}
/// <summary>
@@ -624,7 +647,8 @@ namespace SabreTools.Library.DatItems
|| !_sha1.IsNullOrEmpty()
|| !_sha256.IsNullOrEmpty()
|| !_sha384.IsNullOrEmpty()
|| !_sha512.IsNullOrEmpty();
|| !_sha512.IsNullOrEmpty()
|| !_spamsum.IsNullOrEmpty();
}
/// <summary>
@@ -651,7 +675,8 @@ namespace SabreTools.Library.DatItems
&& ConditionalHashEquals(_sha1, other._sha1)
&& ConditionalHashEquals(_sha256, other._sha256)
&& ConditionalHashEquals(_sha384, other._sha384)
&& ConditionalHashEquals(_sha512, other._sha512);
&& ConditionalHashEquals(_sha512, other._sha512)
&& ConditionalHashEquals(_spamsum, other._spamsum);
}
#endregion
@@ -762,6 +787,12 @@ namespace SabreTools.Library.DatItems
if (filter.DatItem_SHA512.MatchesNegativeSet(SHA512) == true)
return false;
// Filter on SpamSum
if (filter.DatItem_SpamSum.MatchesPositiveSet(SpamSum) == false)
return false;
if (filter.DatItem_SpamSum.MatchesNegativeSet(SpamSum) == true)
return false;
// Filter on merge tag
if (filter.DatItem_Merge.MatchesPositiveSet(MergeTag) == false)
return false;
@@ -930,6 +961,9 @@ namespace SabreTools.Library.DatItems
if (fields.Contains(Field.DatItem_SHA512))
SHA512 = null;
if (fields.Contains(Field.DatItem_SpamSum))
SpamSum = null;
if (fields.Contains(Field.DatItem_Merge))
MergeTag = null;
@@ -1058,6 +1092,10 @@ namespace SabreTools.Library.DatItems
key = SHA512;
break;
case Field.DatItem_SpamSum:
key = SpamSum;
break;
// Let the base handle generic stuff
default:
return base.GetKey(bucketedBy, lower, norename);
@@ -1144,6 +1182,12 @@ namespace SabreTools.Library.DatItems
SHA512 = newItem.SHA512;
}
if (fields.Contains(Field.DatItem_SpamSum))
{
if (string.IsNullOrEmpty(SpamSum) && !string.IsNullOrEmpty(newItem.SpamSum))
SpamSum = newItem.SpamSum;
}
if (fields.Contains(Field.DatItem_Merge))
MergeTag = newItem.MergeTag;

58
SabreTools.Library/External/Aaru/IChecksum.cs vendored Normal file
View File

@@ -0,0 +1,58 @@
// /***************************************************************************
// Aaru Data Preservation Suite
// ----------------------------------------------------------------------------
//
// Filename : IChecksum.cs
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// Component : Checksums.
//
// --[ Description ] ----------------------------------------------------------
//
// Provides an interface for implementing checksums and hashes.
//
// --[ License ] --------------------------------------------------------------
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2020 Natalia Portillo
// ****************************************************************************/
namespace Aaru.CommonTypes.Interfaces
{
public interface IChecksum
{
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of buffer to hash.</param>
void Update(byte[] data, uint len);
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
void Update(byte[] data);
/// <summary>Returns a byte array of the hash value.</summary>
byte[] Final();
/// <summary>Returns a hexadecimal representation of the hash value.</summary>
string End();
}
}

531
SabreTools.Library/External/Aaru/SpamSumContext.cs vendored Normal file
View File

@@ -0,0 +1,531 @@
// /***************************************************************************
// Aaru Data Preservation Suite
// ----------------------------------------------------------------------------
//
// Filename : SpamSumContext.cs
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// Component : Checksums.
//
// --[ Description ] ----------------------------------------------------------
//
// Implements the SpamSum fuzzy hashing algorithm.
//
// --[ License ] --------------------------------------------------------------
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2020 Natalia Portillo
// ****************************************************************************/
// Based on ssdeep
// Copyright (C) 2002 Andrew Tridgell <tridge@samba.org>
// Copyright (C) 2006 ManTech International Corporation
// Copyright (C) 2013 Helmut Grohne <helmut@subdivi.de>
//
// Earlier versions of this code were named fuzzy.c and can be found at:
// http://www.samba.org/ftp/unpacked/junkcode/spamsum/
// http://ssdeep.sf.net/
using System;
using System.Runtime.CompilerServices;
using System.Text;
using Aaru.CommonTypes.Interfaces;
namespace Aaru.Checksums
{
/// <summary>Implements the SpamSum fuzzy hashing algorithm.</summary>
public sealed class SpamSumContext : IChecksum, IDisposable
{
const uint ROLLING_WINDOW = 7;
const uint MIN_BLOCKSIZE = 3;
const uint HASH_PRIME = 0x01000193;
const uint HASH_INIT = 0x28021967;
const uint NUM_BLOCKHASHES = 31;
const uint SPAMSUM_LENGTH = 64;
const uint FUZZY_MAX_RESULT = (2 * SPAMSUM_LENGTH) + 20;
//"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
readonly byte[] _b64 =
{
0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52,
0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A,
0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x30, 0x31,
0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2B, 0x2F
};
FuzzyState _self;
/// <summary>Initializes the SpamSum structures</summary>
public SpamSumContext()
{
_self = new FuzzyState
{
Bh = new BlockhashContext[NUM_BLOCKHASHES]
};
for(int i = 0; i < NUM_BLOCKHASHES; i++)
_self.Bh[i].Digest = new byte[SPAMSUM_LENGTH];
_self.Bhstart = 0;
_self.Bhend = 1;
_self.Bh[0].H = HASH_INIT;
_self.Bh[0].Halfh = HASH_INIT;
_self.Bh[0].Digest[0] = 0;
_self.Bh[0].Halfdigest = 0;
_self.Bh[0].Dlen = 0;
_self.TotalSize = 0;
roll_init();
}
/// <inheritdoc />
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of buffer to hash.</param>
public void Update(byte[] data, uint len)
{
_self.TotalSize += len;
for(int i = 0; i < len; i++)
fuzzy_engine_step(data[i]);
}
/// <inheritdoc />
/// <summary>Updates the hash with data.</summary>
/// <param name="data">Data buffer.</param>
public void Update(byte[] data) => Update(data, (uint)data.Length);
/// <inheritdoc />
/// <summary>Returns a byte array of the hash value.</summary>
public byte[] Final() => throw new NotImplementedException("SpamSum does not have a binary representation.");
/// <inheritdoc />
/// <summary>Returns a base64 representation of the hash value.</summary>
public string End()
{
FuzzyDigest(out byte[] result);
return CToString(result);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void roll_init() => _self.Roll = new RollState
{
Window = new byte[ROLLING_WINDOW]
};
/*
* a rolling hash, based on the Adler checksum. By using a rolling hash
* we can perform auto resynchronisation after inserts/deletes
* internally, h1 is the sum of the bytes in the window and h2
* is the sum of the bytes times the index
* h3 is a shift/xor based rolling hash, and is mostly needed to ensure that
* we can cope with large blocksize values
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void roll_hash(byte c)
{
_self.Roll.H2 -= _self.Roll.H1;
_self.Roll.H2 += ROLLING_WINDOW * c;
_self.Roll.H1 += c;
_self.Roll.H1 -= _self.Roll.Window[_self.Roll.N % ROLLING_WINDOW];
_self.Roll.Window[_self.Roll.N % ROLLING_WINDOW] = c;
_self.Roll.N++;
/* The original spamsum AND'ed this value with 0xFFFFFFFF which
* in theory should have no effect. This AND has been removed
* for performance (jk) */
_self.Roll.H3 <<= 5;
_self.Roll.H3 ^= c;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
uint roll_sum() => _self.Roll.H1 + _self.Roll.H2 + _self.Roll.H3;
/* A simple non-rolling hash, based on the FNV hash. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static uint sum_hash(byte c, uint h) => (h * HASH_PRIME) ^ c;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static uint SSDEEP_BS(uint index) => MIN_BLOCKSIZE << (int)index;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_try_fork_blockhash()
{
if(_self.Bhend >= NUM_BLOCKHASHES)
return;
if(_self.Bhend == 0) // assert
throw new Exception("Assertion failed");
uint obh = _self.Bhend - 1;
uint nbh = _self.Bhend;
_self.Bh[nbh].H = _self.Bh[obh].H;
_self.Bh[nbh].Halfh = _self.Bh[obh].Halfh;
_self.Bh[nbh].Digest[0] = 0;
_self.Bh[nbh].Halfdigest = 0;
_self.Bh[nbh].Dlen = 0;
++_self.Bhend;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_try_reduce_blockhash()
{
if(_self.Bhstart >= _self.Bhend)
throw new Exception("Assertion failed");
if(_self.Bhend - _self.Bhstart < 2)
/* Need at least two working hashes. */
return;
if((ulong)SSDEEP_BS(_self.Bhstart) * SPAMSUM_LENGTH >= _self.TotalSize)
/* Initial blocksize estimate would select this or a smaller
* blocksize. */
return;
if(_self.Bh[_self.Bhstart + 1].Dlen < SPAMSUM_LENGTH / 2)
/* Estimate adjustment would select this blocksize. */
return;
/* At this point we are clearly no longer interested in the
* start_blocksize. Get rid of it. */
++_self.Bhstart;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void fuzzy_engine_step(byte c)
{
uint i;
/* At each character we update the rolling hash and the normal hashes.
* When the rolling hash hits a reset value then we emit a normal hash
* as a element of the signature and reset the normal hash. */
roll_hash(c);
ulong h = roll_sum();
for(i = _self.Bhstart; i < _self.Bhend; ++i)
{
_self.Bh[i].H = sum_hash(c, _self.Bh[i].H);
_self.Bh[i].Halfh = sum_hash(c, _self.Bh[i].Halfh);
}
for(i = _self.Bhstart; i < _self.Bhend; ++i)
{
/* With growing blocksize almost no runs fail the next test. */
if(h % SSDEEP_BS(i) != SSDEEP_BS(i) - 1)
/* Once this condition is false for one bs, it is
* automatically false for all further bs. I.e. if
* h === -1 (mod 2*bs) then h === -1 (mod bs). */
break;
/* We have hit a reset point. We now emit hashes which are
* based on all characters in the piece of the message between
* the last reset point and this one */
if(0 == _self.Bh[i].Dlen)
fuzzy_try_fork_blockhash();
_self.Bh[i].Digest[_self.Bh[i].Dlen] = _b64[_self.Bh[i].H % 64];
_self.Bh[i].Halfdigest = _b64[_self.Bh[i].Halfh % 64];
if(_self.Bh[i].Dlen < SPAMSUM_LENGTH - 1)
{
/* We can have a problem with the tail overflowing. The
* easiest way to cope with this is to only reset the
* normal hash if we have room for more characters in
* our signature. This has the effect of combining the
* last few pieces of the message into a single piece
* */
_self.Bh[i].Digest[++_self.Bh[i].Dlen] = 0;
_self.Bh[i].H = HASH_INIT;
if(_self.Bh[i].Dlen >= SPAMSUM_LENGTH / 2)
continue;
_self.Bh[i].Halfh = HASH_INIT;
_self.Bh[i].Halfdigest = 0;
}
else
fuzzy_try_reduce_blockhash();
}
}
// CLAUNIA: Flags seems to never be used in ssdeep, so I just removed it for code simplicity
[MethodImpl(MethodImplOptions.AggressiveInlining)]
void FuzzyDigest(out byte[] result)
{
var sb = new StringBuilder();
uint bi = _self.Bhstart;
uint h = roll_sum();
int remain = (int)(FUZZY_MAX_RESULT - 1); /* Exclude terminating '\0'. */
result = new byte[FUZZY_MAX_RESULT];
/* Verify that our elimination was not overeager. */
if(!(bi == 0 || ((ulong)SSDEEP_BS(bi) / 2) * SPAMSUM_LENGTH < _self.TotalSize))
throw new Exception("Assertion failed");
int resultOff = 0;
/* Initial blocksize guess. */
while((ulong)SSDEEP_BS(bi) * SPAMSUM_LENGTH < _self.TotalSize)
{
++bi;
if(bi >= NUM_BLOCKHASHES)
throw new OverflowException("The input exceeds data types.");
}
/* Adapt blocksize guess to actual digest length. */
while(bi >= _self.Bhend)
--bi;
while(bi > _self.Bhstart &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
--bi;
if(bi > 0 &&
_self.Bh[bi].Dlen < SPAMSUM_LENGTH / 2)
throw new Exception("Assertion failed");
sb.AppendFormat("{0}:", SSDEEP_BS(bi));
int i = Encoding.ASCII.GetBytes(sb.ToString()).Length;
if(i <= 0)
/* Maybe snprintf has set errno here? */
throw new OverflowException("The input exceeds data types.");
if(i >= remain)
throw new Exception("Assertion failed");
remain -= i;
Array.Copy(Encoding.ASCII.GetBytes(sb.ToString()), 0, result, 0, i);
resultOff = i;
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
if(h != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _b64[_self.Bh[bi].H % 64];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
else if(_self.Bh[bi].Digest[i] != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = _self.Bh[bi].Digest[i];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = 0x3A; // ':'
--remain;
if(bi < _self.Bhend - 1)
{
++bi;
i = (int)_self.Bh[bi].Dlen;
if(i > remain)
throw new Exception("Assertion failed");
Array.Copy(_self.Bh[bi].Digest, 0, result, resultOff, i);
resultOff += i;
remain -= i;
if(h != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
h = _self.Bh[bi].Halfh;
result[resultOff] = _b64[h % 64];
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
else
{
i = _self.Bh[bi].Halfdigest;
if(i != 0)
{
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff] = (byte)i;
if(i < 3 ||
result[resultOff] != result[resultOff - 1] ||
result[resultOff] != result[resultOff - 2] ||
result[resultOff] != result[resultOff - 3])
{
++resultOff;
--remain;
}
}
}
}
else if(h != 0)
{
if(_self.Bh[bi].Dlen != 0)
throw new Exception("Assertion failed");
if(remain <= 0)
throw new Exception("Assertion failed");
result[resultOff++] = _b64[_self.Bh[bi].H % 64];
/* No need to bother with FUZZY_FLAG_ELIMSEQ, because this
* digest has length 1. */
--remain;
}
result[resultOff] = 0;
}
/// <summary>Gets the hash of a file</summary>
/// <param name="filename">File path.</param>
public static byte[] File(string filename) =>
throw new NotImplementedException("SpamSum does not have a binary representation.");
/// <summary>Gets the hash of a file in hexadecimal and as a byte array.</summary>
/// <param name="filename">File path.</param>
/// <param name="hash">Byte array of the hash value.</param>
public static string File(string filename, out byte[] hash) =>
throw new NotImplementedException("Not yet implemented.");
/// <summary>Gets the hash of the specified data buffer.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="len">Length of the data buffer to hash.</param>
/// <param name="hash">null</param>
/// <returns>Base64 representation of SpamSum $blocksize:$hash:$hash</returns>
public static string Data(byte[] data, uint len, out byte[] hash)
{
var fuzzyContext = new SpamSumContext();
fuzzyContext.Update(data, len);
hash = null;
return fuzzyContext.End();
}
/// <summary>Gets the hash of the specified data buffer.</summary>
/// <param name="data">Data buffer.</param>
/// <param name="hash">null</param>
/// <returns>Base64 representation of SpamSum $blocksize:$hash:$hash</returns>
public static string Data(byte[] data, out byte[] hash) => Data(data, (uint)data.Length, out hash);
// Converts an ASCII null-terminated string to .NET string
[MethodImpl(MethodImplOptions.AggressiveInlining)]
static string CToString(byte[] cString)
{
int count = 0;
// ReSharper disable once LoopCanBeConvertedToQuery
// LINQ is six times slower
foreach(byte c in cString)
{
if(c == 0)
break;
count++;
}
return Encoding.ASCII.GetString(cString, 0, count);
}
public void Dispose()
{
// TODO: No-op to dispose for now
}
struct RollState
{
public byte[] Window;
// ROLLING_WINDOW
public uint H1;
public uint H2;
public uint H3;
public uint N;
}
/* A blockhash contains a signature state for a specific (implicit) blocksize.
* The blocksize is given by SSDEEP_BS(index). The h and halfh members are the
* FNV hashes, where halfh stops to be reset after digest is SPAMSUM_LENGTH/2
* long. The halfh hash is needed be able to truncate digest for the second
* output hash to stay compatible with ssdeep output. */
struct BlockhashContext
{
public uint H;
public uint Halfh;
public byte[] Digest;
// SPAMSUM_LENGTH
public byte Halfdigest;
public uint Dlen;
}
struct FuzzyState
{
public uint Bhstart;
public uint Bhend;
public BlockhashContext[] Bh;
//NUM_BLOCKHASHES
public ulong TotalSize;
public RollState Roll;
}
}
}

8
SabreTools.Library/FileTypes/AaruFormat.cs
View File

@@ -38,12 +38,6 @@ namespace SabreTools.Library.FileTypes
#endregion
#region Hashes
// TODO: Support SpamSum
#endregion
#endregion // Private instance variables
#region Constructors
@@ -204,7 +198,7 @@ namespace SabreTools.Library.FileTypes
aif.SHA256 = checksumEntry.checksum;
break;
case AaruChecksumAlgorithm.SpamSum:
// TODO: Support SpamSum
aif.SpamSum = checksumEntry.checksum;
break;
}
}

3
SabreTools.Library/FileTypes/BaseFile.cs
View File

@@ -23,6 +23,7 @@ namespace SabreTools.Library.FileTypes
public byte[] SHA256 { get; set; }
public byte[] SHA384 { get; set; }
public byte[] SHA512 { get; set; }
public byte[] SpamSum { get; set; }
#endregion
@@ -60,6 +61,7 @@ namespace SabreTools.Library.FileTypes
this.SHA256 = temp.SHA256;
this.SHA384 = temp.SHA384;
this.SHA512 = temp.SHA512;
this.SpamSum = temp.SpamSum;
}
}
}
@@ -90,6 +92,7 @@ namespace SabreTools.Library.FileTypes
this.SHA256 = temp.SHA256;
this.SHA384 = temp.SHA384;
this.SHA512 = temp.SHA512;
this.SpamSum = temp.SpamSum;
}
}

5
SabreTools.Library/Filtering/Filter.cs
View File

@@ -114,6 +114,7 @@ namespace SabreTools.Library.Filtering
public FilterItem<string> DatItem_SHA256 { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_SHA384 { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_SHA512 { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_SpamSum { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_Merge { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_Region { get; private set; } = new FilterItem<string>();
public FilterItem<string> DatItem_Offset { get; private set; } = new FilterItem<string>();
@@ -584,6 +585,10 @@ namespace SabreTools.Library.Filtering
SetStringFilter(DatItem_SHA512, value, negate);
break;
case Field.DatItem_SpamSum:
SetStringFilter(DatItem_SpamSum, value, negate);
break;
case Field.DatItem_Merge:
SetStringFilter(DatItem_Merge, value, negate);
break;

2
SabreTools.Library/IO/FileExtensions.cs
View File

@@ -91,6 +91,8 @@ namespace SabreTools.Library.IO
return DatFormat.RedumpSHA384;
case "sha512":
return DatFormat.RedumpSHA512;
case "spamsum":
return DatFormat.RedumpSpamSum;
case "ssv":
return DatFormat.SSV;
case "tsv":

3
SabreTools.Library/IO/StreamExtensions.cs
View File

@@ -118,6 +118,7 @@ namespace SabreTools.Library.IO
new Hasher(Hash.SHA256),
new Hasher(Hash.SHA384),
new Hasher(Hash.SHA512),
new Hasher(Hash.SpamSum),
};
// Initialize the hashing helpers
@@ -184,6 +185,8 @@ namespace SabreTools.Library.IO
rom.SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash();
if (!omitFromScan.HasFlag(Hash.SHA512))
rom.SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash();
if (!omitFromScan.HasFlag(Hash.SpamSum))
rom.SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash();
// Dispose of the hashers
loadBuffer.Dispose();

4
SabreTools.Library/README.1ST
View File

@@ -230,6 +230,10 @@ Options:
This allows the user to skip calculating the SHA-512 for each of the
files which will speed up the creation of the DAT.
--nss, --skip-spamsum Include SpamSum in output
This allows the user to skip calculating the SpamSum for each of the
files which will speed up the creation of the DAT.
-b, --no-automatic-date Don't include date in file name
Normally, the DAT will be created with the date in the file name in
brackets. This flag removes that instead of the default.

12
SabreTools.Library/Tools/Converters.cs
View File

@@ -34,6 +34,8 @@ namespace SabreTools.Library.Tools
return Field.DatItem_SHA384;
case Hash.SHA512:
return Field.DatItem_SHA512;
case Hash.SpamSum:
return Field.DatItem_SpamSum;
default:
return Field.NULL;
@@ -160,6 +162,8 @@ namespace SabreTools.Library.Tools
case "smdb":
case "everdrive":
return DatFormat.EverdriveSMDB;
case "spamsum":
return DatFormat.RedumpSpamSum;
case "ssv":
return DatFormat.SSV;
case "tsv":
@@ -789,6 +793,10 @@ namespace SabreTools.Library.Tools
case "sha_512_hash":
return Field.DatItem_SHA512;
case "spamsum":
case "spam_sum":
return Field.DatItem_SpamSum;
case "merge":
case "mergetag":
case "merge_tag":
@@ -1478,6 +1486,10 @@ namespace SabreTools.Library.Tools
case "sha_512_hash":
return Field.DatItem_SHA512;
case "spamsum":
case "spam_sum":
return Field.DatItem_SpamSum;
case "merge":
case "mergetag":
case "merge_tag":

17
SabreTools.Library/Tools/Hasher.cs
View File

@@ -2,7 +2,7 @@
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Aaru.Checksums;
using SabreTools.Library.DatFiles;
using SabreTools.Library.External;
@@ -58,6 +58,10 @@ namespace SabreTools.Library.Tools
case Hash.SHA512:
_hasher = SHA512.Create();
break;
case Hash.SpamSum:
_hasher = new SpamSumContext();
break;
}
}
@@ -87,6 +91,10 @@ namespace SabreTools.Library.Tools
case Hash.SHA512:
await Task.Run(() => (_hasher as HashAlgorithm).TransformBlock(buffer, 0, size, null, 0));
break;
case Hash.SpamSum:
await Task.Run(() => (_hasher as SpamSumContext).Update(buffer));
break;
}
}
@@ -112,6 +120,10 @@ namespace SabreTools.Library.Tools
case Hash.SHA512:
await Task.Run(() => (_hasher as HashAlgorithm).TransformFinalBlock(emptyBuffer, 0, 0));
break;
case Hash.SpamSum:
// No finalization step needed
break;
}
}
@@ -134,6 +146,9 @@ namespace SabreTools.Library.Tools
case Hash.SHA384:
case Hash.SHA512:
return (_hasher as HashAlgorithm).Hash;
case Hash.SpamSum:
return (_hasher as SpamSumContext).Final();
}
return null;

2
SabreTools.Library/Tools/Sanitizer.cs
View File

@@ -145,7 +145,7 @@ namespace SabreTools.Library.Tools
/// <param name="hash">Hash string to sanitize</param>
/// <param name="padding">Amount of characters to pad to</param>
/// <returns>Cleaned string</returns>
private static string CleanHashData(string hash, int padding)
public static string CleanHashData(string hash, int padding)
{
// If we have a known blank hash, return blank
if (string.IsNullOrWhiteSpace(hash) || hash == "-" || hash == "_")

17
SabreTools/Features/BaseFeature.cs
View File

@@ -1020,6 +1020,20 @@ namespace SabreTools.Features
}
}
internal const string SkipSpamSumValue = "skip-spamsum";
internal static Library.Help.Feature SkipSpamSumFlag
{
get
{
return new Library.Help.Feature(
SkipSpamSumValue,
new List<string>() { "-nss", "--skip-spamsum" },
"Include SpamSum in output", // TODO: This needs to be inverted later
Library.Help.FeatureType.Flag,
longDescription: "This allows the user to skip calculating the SpamSum for each of the files which will speed up the creation of the DAT.");
}
}
internal const string SuperdatValue = "superdat";
internal static Library.Help.Feature SuperdatFlag
{
@@ -2462,6 +2476,8 @@ Some special strings that can be used:
omitFromScan &= ~Hash.SHA384; // TODO: This needs to be inverted later
if (GetBoolean(features, SkipSha512Value))
omitFromScan &= ~Hash.SHA512; // TODO: This needs to be inverted later
if (GetBoolean(features, SkipSpamSumValue))
omitFromScan &= ~Hash.SpamSum; // TODO: This needs to be inverted later
return omitFromScan;
}
@@ -2611,6 +2627,7 @@ Some special strings that can be used:
updateFields.Add(Field.DatItem_SHA256);
updateFields.Add(Field.DatItem_SHA384);
updateFields.Add(Field.DatItem_SHA512);
updateFields.Add(Field.DatItem_SpamSum);
}
if (GetBoolean(features, UpdateManufacturerValue))

1
SabreTools/Features/DatFromDir.cs
View File

@@ -29,6 +29,7 @@ namespace SabreTools.Features
AddFeature(SkipSha256Flag);
AddFeature(SkipSha384Flag);
AddFeature(SkipSha512Flag);
AddFeature(SkipSpamSumFlag);
AddFeature(NoAutomaticDateFlag);
AddFeature(AaruFormatsAsFilesFlag);