32 Commits
1.4.2 ... 1.5.1

Author SHA1 Message Date
Matt Nadareski
458499eea4 Bump version 2025-10-07 09:06:58 -04:00
Matt Nadareski
47cb600f3f Fix accidental cutoff 2025-10-07 09:02:11 -04:00
Matt Nadareski
d42af9b95d Consistency matching cleanup 2025-10-07 09:01:12 -04:00
Matt Nadareski
8672fea581 Used mixed features and inputs 2025-10-06 11:31:54 -04:00
Matt Nadareski
a5b36a8329 Update readme to match help text 2025-10-06 09:12:58 -04:00
Matt Nadareski
b83cafebf1 Reorganize based on lessons from other implementations 2025-10-06 09:12:48 -04:00
Matt Nadareski
2b7f09f06f Update CommandLine to 1.3.2 2025-10-06 07:29:48 -04:00
Matt Nadareski
2a37f8c03a Set text formatting in readme 2025-10-05 22:08:02 -04:00
Matt Nadareski
51c3ecec4b Fully use CommandLine modelling 2025-10-05 20:31:15 -04:00
Matt Nadareski
97a1338cab Update CommandLine to 1.3.1 2025-10-05 20:11:35 -04:00
Matt Nadareski
a808c9ba3c Use CommandLine library for executable 2025-10-05 19:47:02 -04:00
Matt Nadareski
53e28df7f8 Make MekaCrc official 2025-09-25 07:51:44 -04:00
Matt Nadareski
bdde3bad16 Use BitConverter instead of Convert 2025-09-25 07:39:57 -04:00
Matt Nadareski
32491ccaae Add "mekacrc", not hooked up or validated 2025-09-24 23:20:30 -04:00
Matt Nadareski
ae3fc9ef56 Fix typo introduced by cleanup 2025-09-24 22:38:38 -04:00
Matt Nadareski
8d52187a03 Minor typing cleanup 2025-09-24 22:36:45 -04:00
Matt Nadareski
720a3a4c2b Slightly improve HasCommonSubstring 2025-09-24 22:35:40 -04:00
Matt Nadareski
0872d8c927 Slight reorganization 2025-09-24 22:20:31 -04:00
Matt Nadareski
1619046d11 Rewrite comparison code to fit C# better, add tests 2025-09-24 22:18:02 -04:00
Matt Nadareski
4fda55d4d1 Move static method higher in the class 2025-09-24 21:58:12 -04:00
Matt Nadareski
349a414ff3 Slight cleanup based on preferred style and naming 2025-09-24 21:55:20 -04:00
Matt Nadareski
05617c5c7e Slight test cleanup 2025-09-24 21:41:26 -04:00
HeroponRikiBestest
cce8a18b03 Add SpamSum fuzzy compare (#3)
* Pre-cleaned drop in fuzzycompare

* Finish cleaning up code.

* Figure out why i can't debug my unit tests

* Fix stupid mistake

* First round of changes

* Extra tests

* Revert TestHelper.cs

* Roll back TestHelper.cs correctly.

* Roll back Options.cs and Program.cs
2025-09-24 21:32:11 -04:00
Matt Nadareski
cbaa79b284 Minor tweaks 2025-09-23 14:18:50 -04:00
Matt Nadareski
32cd0e73ed Add hashing tool implementation, for fun 2025-09-23 14:15:27 -04:00
Matt Nadareski
95589df904 There 2025-09-10 21:52:28 -04:00
Matt Nadareski
fd612f939a Bump version 2025-07-24 08:44:08 -04:00
Matt Nadareski
8e3a0f77e9 Be consistent about end-of-file newlines 2025-07-24 08:41:37 -04:00
Matt Nadareski
40cfa78be4 Add .NET Standard 2.0 and 2.1 2025-07-24 08:36:34 -04:00
Matt Nadareski
4d92c7cd23 Update nuget packages 2025-07-24 08:34:59 -04:00
Matt Nadareski
acf1e3ec71 Reduce target frameworks for test project 2025-02-25 21:28:49 -05:00
Matt Nadareski
10027f78e3 Fix how conditions are used for references 2025-02-25 21:24:38 -05:00
80 changed files with 1519 additions and 107 deletions

View File

@@ -25,13 +25,13 @@ jobs:
run: dotnet test
- name: Run publish script
run: ./publish-nix.sh
run: ./publish-nix.sh -d
- name: Upload to rolling
uses: ncipollo/release-action@v1.14.0
with:
allowUpdates: True
artifacts: "*.nupkg,*.snupkg"
artifacts: "*.nupkg,*.snupkg,*.zip"
body: 'Last built commit: ${{ github.sha }}'
name: 'Rolling Release'
prerelease: True

28
.vscode/launch.json vendored Normal file
View File

@@ -0,0 +1,28 @@
{
// Use IntelliSense to find out which attributes exist for C# debugging
// Use hover for the description of the existing attributes
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Launch",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/Hasher/bin/Debug/net9.0/Hasher.dll",
"args": [],
"cwd": "${workspaceFolder}",
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
"console": "internalConsole",
"stopAtEntry": false,
"justMyCode": false
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}"
}
]
}

View File

@@ -0,0 +1,52 @@
using System;
using SabreTools.CommandLine;
using SabreTools.Hashing;
namespace Hasher.Features
{
internal sealed class ListFeature : Feature
{
#region Feature Definition
public const string DisplayName = "list";
private static readonly string[] _flags = ["--list"];
private const string _description = "List all available hashes and quit";
#endregion
public ListFeature()
: base(DisplayName, _flags, _description)
{
RequiresInputs = false;
}
/// <inheritdoc/>
/// TODO: Print all supported variants of names?
public override bool Execute()
{
Console.WriteLine("Hash Name Parameter Name ");
Console.WriteLine("--------------------------------------------------------------");
var hashTypes = (HashType[])Enum.GetValues(typeof(HashType));
foreach (var hashType in hashTypes)
{
// Derive the parameter name
string paramName = $"{hashType}";
paramName = paramName.Replace("-", string.Empty);
paramName = paramName.Replace(" ", string.Empty);
paramName = paramName.Replace("/", "_");
paramName = paramName.Replace("\\", "_");
paramName = paramName.ToLowerInvariant();
Console.WriteLine($"{hashType.GetHashName()?.PadRight(39, ' ')} {paramName}");
}
return true;
}
/// <inheritdoc/>
public override bool VerifyInputs() => true;
}
}

View File

@@ -0,0 +1,170 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using SabreTools.CommandLine;
using SabreTools.CommandLine.Inputs;
using SabreTools.Hashing;
namespace Hasher.Features
{
internal sealed class MainFeature : Feature
{
#region Feature Definition
public const string DisplayName = "main";
/// <remarks>Flags are unused</remarks>
private static readonly string[] _flags = [];
/// <remarks>Description is unused</remarks>
private const string _description = "";
#endregion
#region Inputs
private const string _debugName = "debug";
internal readonly FlagInput DebugInput = new(_debugName, ["-d", "--debug"], "Enable debug mode");
private const string _typeName = "type";
internal readonly StringListInput TypeInput = new(_typeName, ["-t", "--type"], "Select included hashes");
#endregion
public MainFeature()
: base(DisplayName, _flags, _description)
{
RequiresInputs = true;
Add(DebugInput);
Add(TypeInput);
}
/// <inheritdoc/>
public override bool Execute()
{
// Get the required variables
bool debug = GetBoolean(_debugName);
List<HashType> hashTypes = GetHashTypes(GetStringList(_typeName));
// Loop through all of the input files
for (int i = 0; i < Inputs.Count; i++)
{
string arg = Inputs[i];
PrintPathHashes(arg, hashTypes, debug);
}
return true;
}
/// <inheritdoc/>
public override bool VerifyInputs() => true;
/// <summary>
/// Derive a list of hash types from a list of strings
/// </summary>
private static List<HashType> GetHashTypes(List<string> types)
{
List<HashType> hashTypes = [];
if (types.Count == 0)
{
hashTypes.Add(HashType.CRC32);
hashTypes.Add(HashType.MD5);
hashTypes.Add(HashType.SHA1);
hashTypes.Add(HashType.SHA256);
}
else if (types.Contains("all"))
{
hashTypes = [.. (HashType[])Enum.GetValues(typeof(HashType))];
}
else
{
foreach (string typeString in types)
{
HashType? hashType = typeString.GetHashType();
if (hashType != null && !hashTypes.Contains(hashType.Value))
hashTypes.Add(item: hashType.Value);
}
}
return hashTypes;
}
/// <summary>
/// Wrapper to print hashes for a single path
/// </summary>
/// <param name="path">File or directory path</param>
/// <param name="hashTypes">Set of hashes to retrieve</param>
/// <param name="debug">Enable debug output</param>
private static void PrintPathHashes(string path, List<HashType> hashTypes, bool debug)
{
Console.WriteLine($"Checking possible path: {path}");
// Check if the file or directory exists
if (File.Exists(path))
{
PrintFileHashes(path, hashTypes, debug);
}
else if (Directory.Exists(path))
{
foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories))
{
PrintFileHashes(file, hashTypes, debug);
}
}
else
{
Console.WriteLine($"{path} does not exist, skipping...");
}
}
/// <summary>
/// Print information for a single file, if possible
/// </summary>
/// <param name="file">File path</param>
/// <param name="hashTypes">Set of hashes to retrieve</param>
/// <param name="debug">Enable debug output</param>
private static void PrintFileHashes(string file, List<HashType> hashTypes, bool debug)
{
Console.WriteLine($"Attempting to hash {file}, this may take a while...");
Console.WriteLine();
// If the file doesn't exist
if (!File.Exists(file))
{
Console.WriteLine($"{file} does not exist, skipping...");
return;
}
try
{
// Get all file hashes for flexibility
var hashes = HashTool.GetFileHashes(file);
if (hashes == null)
{
if (debug) Console.WriteLine($"Hashes for {file} could not be retrieved");
return;
}
// Output subset of available hashes
var builder = new StringBuilder();
foreach (HashType hashType in hashTypes)
{
// TODO: Make helper to pretty-print hash type names
if (hashes.TryGetValue(hashType, out string? hash) && hash != null)
builder.AppendLine($"{hashType}: {hash}");
}
// Create and print the output data
string hashData = builder.ToString();
Console.WriteLine(hashData);
}
catch (Exception ex)
{
Console.WriteLine(debug ? ex : "[Exception opening file, please try again]");
return;
}
}
}
}

37
Hasher/Hasher.csproj Normal file
View File

@@ -0,0 +1,37 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net20;net35;net40;net452;net462;net472;net48;netcoreapp3.1;net5.0;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
<OutputType>Exe</OutputType>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
<IncludeNativeLibrariesForSelfExtract>true</IncludeNativeLibrariesForSelfExtract>
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
<LangVersion>latest</LangVersion>
<Nullable>enable</Nullable>
<SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<Version>1.5.1</Version>
</PropertyGroup>
<!-- Support All Frameworks -->
<PropertyGroup Condition="$(TargetFramework.StartsWith(`net2`)) OR $(TargetFramework.StartsWith(`net3`)) OR $(TargetFramework.StartsWith(`net4`))">
<RuntimeIdentifiers>win-x86;win-x64</RuntimeIdentifiers>
</PropertyGroup>
<PropertyGroup Condition="$(TargetFramework.StartsWith(`netcoreapp`)) OR $(TargetFramework.StartsWith(`net5`))">
<RuntimeIdentifiers>win-x86;win-x64;win-arm64;linux-x64;linux-arm64;osx-x64</RuntimeIdentifiers>
</PropertyGroup>
<PropertyGroup Condition="$(TargetFramework.StartsWith(`net6`)) OR $(TargetFramework.StartsWith(`net7`)) OR $(TargetFramework.StartsWith(`net8`)) OR $(TargetFramework.StartsWith(`net9`))">
<RuntimeIdentifiers>win-x86;win-x64;win-arm64;linux-x64;linux-arm64;osx-x64;osx-arm64</RuntimeIdentifiers>
</PropertyGroup>
<PropertyGroup Condition="$(RuntimeIdentifier.StartsWith(`osx-arm`))">
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\SabreTools.Hashing\SabreTools.Hashing.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="SabreTools.CommandLine" Version="[1.3.2]" />
</ItemGroup>
</Project>

87
Hasher/Program.cs Normal file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using Hasher.Features;
using SabreTools.CommandLine;
using SabreTools.CommandLine.Features;
namespace Hasher
{
public static class Program
{
public static void Main(string[] args)
{
// Create the command set
var mainFeature = new MainFeature();
var commandSet = CreateCommands(mainFeature);
// If we have no args, show the help and quit
if (args == null || args.Length == 0)
{
commandSet.OutputAllHelp();
return;
}
// Cache the first argument and starting index
string featureName = args[0];
// Try processing the standalone arguments
var topLevel = commandSet.GetTopLevel(featureName);
switch (topLevel)
{
// Standalone Options
case Help help: help.ProcessArgs(args, 0, commandSet); return;
case ListFeature lf: lf.Execute(); return;
// Default Behavior
default:
if (!mainFeature.ProcessArgs(args, 0))
{
commandSet.OutputAllHelp();
return;
}
else if (!mainFeature.VerifyInputs())
{
Console.Error.WriteLine("At least one input is required");
commandSet.OutputAllHelp();
return;
}
mainFeature.Execute();
break;
}
}
/// <summary>
/// Create the command set for the program
/// </summary>
private static CommandSet CreateCommands(MainFeature mainFeature)
{
List<string> header = [
"File Hashing Program",
string.Empty,
"Hasher <options> file|directory ...",
string.Empty,
];
List<string> footer = [
string.Empty,
"If no hash types are provided, this tool will default to",
"outputting CRC-32, MD5, SHA-1, and SHA-256.",
"Optionally, all supported hashes can be output by",
"specifying a value of 'all'.",
];
var commandSet = new CommandSet(header, footer);
// Standalone Options
commandSet.Add(new Help(["-?", "-h", "--help"]));
commandSet.Add(new ListFeature());
// Hasher Options
commandSet.Add(mainFeature.DebugInput);
commandSet.Add(mainFeature.TypeInput);
return commandSet;
}
}
}

7
LICENSE Normal file
View File

@@ -0,0 +1,7 @@
Copyright (c) 2018-2025 Matt Nadareski
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@@ -12,6 +12,25 @@ For the most recent stable build, download the latest release here: [Releases Pa
For the latest WIP build here: [Rolling Release](https://github.com/SabreTools/SabreTools.Hashing/releases/rolling)
## Hasher
**Hasher** is a reference implementation for hashing and checksumming features of the library, packaged as a standalone executable for all supported platforms. It will attempt to select the correct hashing types based on input and then print the calculated values to standard output.
```text
Hasher <options> file|directory ...
Available options:
-?, -h, --help Show this help
--list List all available hashes and quit
-d, --debug Enable debug mode
-t=, --type= Select included hashes
If no hash types are provided, this tool will default to
outputting CRC-32, MD5, SHA-1, and SHA-256.
Optionally, all supported hashes can be output by
specifying a value of 'all'.
```
## Internal Implementations
All hash and checksum types here have been written to ensure compatibility across all .NET versions. Some may have been adapted to ensure this compatibility. These can be treated as reference implementations, not always optimized.

View File

@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net462;net472;net48;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
<TargetFrameworks>net8.0;net9.0</TargetFrameworks>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
<IsPackable>false</IsPackable>
<LangVersion>latest</LangVersion>
@@ -20,14 +20,14 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PackageReference Include="coverlet.collector" Version="6.0.4">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="System.IO.Compression" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>

View File

@@ -0,0 +1,45 @@
using Xunit;
namespace SabreTools.Hashing.Test
{
public class SpamSumTests
{
[Theory]
// Invalid inputs
[InlineData(null, null, -1)]
[InlineData(null, "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", -1)]
[InlineData("3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", null, -1)]
[InlineData("", "", -1)]
[InlineData("3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", "", -1)]
[InlineData("", "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", -1)]
// Small data
[InlineData("6:l+lq/MtlM8pJ0gt6lXWogE61UlT1Uqj1akMD5n:l+l6Mtl/n0gtOXmEuUl5UqpakM9n", "6:mTj3qJskr+V+1o21+n0rtD2noPWKlAyjllZmMt6120EK+wlsS6T1oLwXuk4tk7:m/bk/1oQrJL3jTu20EK+wlsp5oO4tk7", 0)]
[InlineData("6:mTj3qJskr+V+1o21+n0rtD2noPWKlAyjllZmMt6120EK+wlsS6T1oLwXuk4tk7:m/bk/1oQrJL3jTu20EK+wlsp5oO4tk7", "6:l+lq/MtlM8pJ0gt6lXWogE61UlT1Uqj1akMD5n:l+l6Mtl/n0gtOXmEuUl5UqpakM9n", 0)]
[InlineData("3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", "3:hMCERJAFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:huRJdxZENFs+rPSromekL", 41)]
[InlineData("3:hMCERJAFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:huRJdxZENFs+rPSromekL", "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", 41)]
[InlineData("12:Y+VH/3Ckg3xqMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMn:xHqVwMMMMMMMMMMMMMMMMMMMMMMMMMM0", "12:Oqkg3xqMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMu:OqVwMMMMMMMMMMMMMMMMMMMMMMMMMMMd", 44)]
[InlineData("12:Oqkg3xqMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMu:OqVwMMMMMMMMMMMMMMMMMMMMMMMMMMMd", "12:Y+VH/3Ckg3xqMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMn:xHqVwMMMMMMMMMMMMMMMMMMMMMMMMMM0", 44)]
// Large data
[InlineData("196608:Gbxf3F4OQK3IuUGM8Ylv1kqCLuDKeo5cRld6iZL6HAGpX7g08WCWDc4NNgs4NEv:qcgxU+UxR2gl5qAGpXjHDcCNgs4N", "196608:EqKRzGWxtDOadbDCbZStQxNy+fox3UgOYorlhjolL0K1WJj5lYA:EbNf76db9xNVox3MRlh+sf", 0)]
[InlineData("196608:EqKRzGWxtDOadbDCbZStQxNy+fox3UgOYorlhjolL0K1WJj5lYA:EbNf76db9xNVox3MRlh+sf", "196608:Gbxf3F4OQK3IuUGM8Ylv1kqCLuDKeo5cRld6iZL6HAGpX7g08WCWDc4NNgs4NEv:qcgxU+UxR2gl5qAGpXjHDcCNgs4N", 0)]
[InlineData("24576:p+QxhkAcV6cUdRxczoy3NmO0ne3HFVjSeQ229SVjeONr+v:YQ/q6baz5Nqe3H2eQzStBa", "24576:fCQxhkAcV6cUdRxczoyVQQFDSVRNihk24vXDj20sq:6Q/q6bazwMgRNihk24jtsq", 54)]
[InlineData("24576:fCQxhkAcV6cUdRxczoyVQQFDSVRNihk24vXDj20sq:6Q/q6bazwMgRNihk24jtsq", "24576:p+QxhkAcV6cUdRxczoy3NmO0ne3HFVjSeQ229SVjeONr+v:YQ/q6baz5Nqe3H2eQzStBa", 54)]
// Duplicate sequence truncation
[InlineData("500:AAAAAAAAAAAAAAAAAAAAAAAAyENFACBE+rW6Tj7SMQmK:4", "500:AAAyENFACBE+rW6Tj7SMQmK:4", 100)]
// Trailing data ignored
[InlineData("6:l+lq/MtlM8pJ0gt6lXWogE61UlT1Uqj1akMD5n:l+l6Mtl/n0gtOXmEuUl5UqpakM9n,ANYTHING", "6:mTj3qJskr+V+1o21+n0rtD2noPWKlAyjllZmMt6120EK+wlsS6T1oLwXuk4tk7:m/bk/1oQrJL3jTu20EK+wlsp5oO4tk7,NOTHING", 0)]
[InlineData("6:mTj3qJskr+V+1o21+n0rtD2noPWKlAyjllZmMt6120EK+wlsS6T1oLwXuk4tk7:m/bk/1oQrJL3jTu20EK+wlsp5oO4tk7,NOTHING", "6:l+lq/MtlM8pJ0gt6lXWogE61UlT1Uqj1akMD5n:l+l6Mtl/n0gtOXmEuUl5UqpakM9n,ANYTHING", 0)]
// Rolling window - larger than 7
[InlineData("500:7SMQmKa:3", "500:7SMQmKr:3", 0)]
// Rolling window - smaller than 7
[InlineData("500:7QmKa:3", "500:7QmKr:3", 0)]
// Blocksize differences
[InlineData("9287:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", "5893:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", 0)]
[InlineData("3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERkL:hZRdxZENFs+rPSromekL", 100)]
public void FuzzyCompareTest(string? stringOne, string? stringTwo, int expected)
{
var result = SpamSum.SpamSum.FuzzyCompare(stringOne, stringTwo);
Assert.Equal(expected, result);
}
}
}

View File

@@ -172,6 +172,8 @@ namespace SabreTools.Hashing.Test
{HashType.FNV1a_32, "9086769b"},
{HashType.FNV1a_64, "399dd1cd965b73db"},
{HashType.MekaCrc, "0a0a0b1174052f22"},
{HashType.MD2, "362e1a6931668e6a9de5c159c52c71b5"},
{HashType.MD4, "61bef59d7a754874fccbd67b4ec2fb10"},
{HashType.MD5, "b722871eaa950016296184d026c5dec9"},

View File

@@ -7,22 +7,56 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.Hashing", "Sabre
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.Hashing.Test", "SabreTools.Hashing.Test\SabreTools.Hashing.Test.csproj", "{A2BCBFDE-685B-4817-B724-050A99E02601}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Hasher", "Hasher\Hasher.csproj", "{5DAC74F2-22AB-409B-B828-2FD3851586A3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|x64.ActiveCfg = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|x64.Build.0 = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|x86.ActiveCfg = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Debug|x86.Build.0 = Debug|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|Any CPU.Build.0 = Release|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|x64.ActiveCfg = Release|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|x64.Build.0 = Release|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|x86.ActiveCfg = Release|Any CPU
{F7E34528-080E-4E60-B9D1-8ADF70A24BB0}.Release|x86.Build.0 = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|x64.ActiveCfg = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|x64.Build.0 = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|x86.ActiveCfg = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Debug|x86.Build.0 = Debug|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|Any CPU.Build.0 = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|x64.ActiveCfg = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|x64.Build.0 = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|x86.ActiveCfg = Release|Any CPU
{A2BCBFDE-685B-4817-B724-050A99E02601}.Release|x86.Build.0 = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|x64.ActiveCfg = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|x64.Build.0 = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|x86.ActiveCfg = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Debug|x86.Build.0 = Debug|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|Any CPU.Build.0 = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|x64.ActiveCfg = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|x64.Build.0 = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|x86.ActiveCfg = Release|Any CPU
{5DAC74F2-22AB-409B-B828-2FD3851586A3}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

View File

@@ -136,9 +136,9 @@ namespace SabreTools.Hashing.Checksum
/// <inheritdoc/>
protected override byte[] HashFinal()
{
byte[] hashArr = BitConverter.GetBytes(_hash);
byte[] hashArr = BitConverter.GetBytes(_hash);
Array.Reverse(hashArr);
return hashArr;
}
}
}
}

View File

@@ -19,4 +19,4 @@ namespace SabreTools.Hashing.Checksum
return bytes;
}
}
}
}

View File

@@ -47,4 +47,4 @@ namespace SabreTools.Hashing.Checksum
return hashArr;
}
}
}
}

View File

@@ -30,4 +30,4 @@ namespace SabreTools.Hashing.Checksum
#endregion
}
}
}

View File

@@ -56,4 +56,4 @@ namespace SabreTools.Hashing.Checksum
return BitOperations.ClampValueToBytes(localHash, Def.Width);
}
}
}
}

View File

@@ -62,4 +62,4 @@ namespace SabreTools.Hashing.Checksum
/// </summary>
public ulong XorOut { get; set; }
}
}
}

View File

@@ -330,4 +330,4 @@ namespace SabreTools.Hashing.Checksum
hash = local;
}
}
}
}

View File

@@ -41,4 +41,4 @@ namespace SabreTools.Hashing.Checksum
_hash = (ushort)((c1 << 8) | c0);
}
}
}
}

View File

@@ -125,4 +125,4 @@ namespace SabreTools.Hashing.Checksum
_hash = (c1 << 16) | c0;
}
}
}
}

View File

@@ -125,4 +125,4 @@ namespace SabreTools.Hashing.Checksum
_hash = (c1 << 32) | c0;
}
}
}
}

View File

@@ -0,0 +1,46 @@
using System;
namespace SabreTools.Hashing.Checksum
{
/// <see href="https://github.com/ocornut/meka/blob/master/meka/srcs/checksum.cpp"/>
public class MekaCrc : ChecksumBase<ulong>
{
/// <inheritdoc/>
public override int HashSize => 64;
public MekaCrc()
{
Initialize();
}
/// <summary>
/// Reset the internal hashing state
/// </summary>
public override void Initialize()
{
_hash = 0;
}
/// <inheritdoc/>
/// <remarks>The original code limits the maximum processed size to 8KiB</remarks>
protected override void HashCore(byte[] data, int offset, int length)
{
// Read the current hash into a byte array
byte[] temp = BitConverter.GetBytes(_hash);
// Loop over the input and process
for (int i = 0; i < length; i++)
{
byte v = data[offset + i];
unchecked
{
temp[v & 7]++;
temp[v >> 5]++;
}
}
// Convert the hash back into a value
_hash = BitConverter.ToUInt64(temp, 0);
}
}
}

View File

@@ -1687,4 +1687,4 @@ namespace SabreTools.Hashing.Checksum
#endregion
}
}
}

View File

@@ -19,4 +19,4 @@ namespace SabreTools.Hashing
#endregion
}
}
}

View File

@@ -694,4 +694,4 @@ namespace SabreTools.Hashing.CryptographicHash
#endregion
}
}
}

View File

@@ -126,7 +126,7 @@ namespace SabreTools.Hashing.CryptographicHash
// Pad the block
byte[] padding = new byte[padLength];
#if NETFRAMEWORK
#if NETFRAMEWORK || NETSTANDARD2_0
for (int i = 0; i < padLength; i++)
{
padding[i] = padLength;
@@ -166,4 +166,4 @@ namespace SabreTools.Hashing.CryptographicHash
}
}
}
}
}

View File

@@ -211,4 +211,4 @@ namespace SabreTools.Hashing.CryptographicHash
/// </summary>
private static uint H(uint x, uint y, uint z) => x ^ y ^ z;
}
}
}

View File

@@ -59,4 +59,4 @@ namespace SabreTools.Hashing.CryptographicHash
Array.Clear(_block, 0, _block.Length);
}
}
}
}

View File

@@ -444,4 +444,4 @@ namespace SabreTools.Hashing.CryptographicHash
/// </summary>
private static uint G48_63(uint x, uint y, uint z) => (x & z) | (y & ~z);
}
}
}

View File

@@ -680,4 +680,4 @@ namespace SabreTools.Hashing.CryptographicHash
/// </summary>
private static uint G64_79(uint x, uint y, uint z) => x ^ (y | ~z);
}
}
}

View File

@@ -464,4 +464,4 @@ namespace SabreTools.Hashing.CryptographicHash
/// </summary>
private static uint G48_63(uint x, uint y, uint z) => (x & z) | (y & ~z);
}
}
}

View File

@@ -705,4 +705,4 @@ namespace SabreTools.Hashing.CryptographicHash
/// </summary>
private static uint G64_79(uint x, uint y, uint z) => x ^ (y | ~z);
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -14,4 +14,4 @@ namespace SabreTools.Hashing.CryptographicHash
_padStart = 0x01;
}
}
}
}

View File

@@ -14,4 +14,4 @@ namespace SabreTools.Hashing.CryptographicHash
_padStart = 0x01;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -25,4 +25,4 @@ namespace SabreTools.Hashing.CryptographicHash
return trimmedHash;
}
}
}
}

View File

@@ -14,4 +14,4 @@ namespace SabreTools.Hashing.CryptographicHash
_padStart = 0x80;
}
}
}
}

View File

@@ -14,4 +14,4 @@ namespace SabreTools.Hashing.CryptographicHash
_padStart = 0x80;
}
}
}
}

View File

@@ -234,4 +234,4 @@ namespace SabreTools.Hashing.CryptographicHash
_block[7] -= _block[6] ^ 0x0123456789ABCDEF;
}
}
}
}

View File

@@ -0,0 +1,9 @@
#if NET20
namespace System.Runtime.CompilerServices
{
[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method)]
internal sealed class ExtensionAttribute : Attribute {}
}
#endif

View File

@@ -0,0 +1,445 @@
namespace SabreTools.Hashing
{
public static class Extensions
{
/// <summary>
/// Get the name of a given hash type, if possible
/// </summary>
/// TODO: This should be automated instead of hardcoded
public static string? GetHashName(this HashType hashType)
{
return hashType switch
{
HashType.Adler32 => "Mark Adler's 32-bit checksum",
#if NET7_0_OR_GREATER
HashType.BLAKE3 => "BLAKE3 512-bit digest",
#endif
HashType.CRC1_ZERO => "CRC-1/ZERO [Parity bit with 0 start]",
HashType.CRC1_ONE => "CRC-1/ONE [Parity bit with 1 start]",
HashType.CRC3_GSM => "CRC-3/GSM",
HashType.CRC3_ROHC => "CRC-3/ROHC",
HashType.CRC4_G704 => "CRC-4/G-704 [CRC-4/ITU]",
HashType.CRC4_INTERLAKEN => "CRC-4/INTERLAKEN",
HashType.CRC5_EPCC1G2 => "CRC-5/EPC-C1G2 [CRC-5/EPC]",
HashType.CRC5_G704 => "CRC-5/G-704 [CRC-5/ITU]",
HashType.CRC5_USB => "CRC-5/USB",
HashType.CRC6_CDMA2000A => "CRC-6/CDMA2000-A",
HashType.CRC6_CDMA2000B => "CRC-6/CDMA2000-B",
HashType.CRC6_DARC => "CRC-6/DARC",
HashType.CRC6_G704 => "CRC-6/G-704 [CRC-6/ITU]",
HashType.CRC6_GSM => "CRC-6/GSM",
HashType.CRC7_MMC => "CRC-7/MMC [CRC-7]",
HashType.CRC7_ROHC => "CRC-7/ROHC",
HashType.CRC7_UMTS => "CRC-7/UMTS",
HashType.CRC8 => "CRC-8",
HashType.CRC8_AUTOSAR => "CRC-8/AUTOSAR",
HashType.CRC8_BLUETOOTH => "CRC-8/BLUETOOTH",
HashType.CRC8_CDMA2000 => "CRC-8/CDMA2000",
HashType.CRC8_DARC => "CRC-8/DARC",
HashType.CRC8_DVBS2 => "CRC-8/DVB-S2",
HashType.CRC8_GSMA => "CRC-8/GSM-A",
HashType.CRC8_GSMB => "CRC-8/GSM-B",
HashType.CRC8_HITAG => "CRC-8/HITAG",
HashType.CRC8_I4321 => "CRC-8/I-432-1 [CRC-8/ITU]",
HashType.CRC8_ICODE => "CRC-8/I-CODE",
HashType.CRC8_LTE => "CRC-8/LTE",
HashType.CRC8_MAXIMDOW => "CRC-8/MAXIM-DOW [CRC-8/MAXIM, DOW-CRC]",
HashType.CRC8_MIFAREMAD => "CRC-8/MIFARE-MAD",
HashType.CRC8_NRSC5 => "CRC-8/NRSC-5",
HashType.CRC8_OPENSAFETY => "CRC-8/OPENSAFETY",
HashType.CRC8_ROHC => "CRC-8/ROHC",
HashType.CRC8_SAEJ1850 => "CRC-8/SAE-J1850",
HashType.CRC8_SMBUS => "CRC-8/SMBUS [CRC-8]",
HashType.CRC8_TECH3250 => "CRC-8/TECH-3250 [CRC-8/AES, CRC-8/EBU]",
HashType.CRC8_WCDMA => "CRC-8/WCDMA",
HashType.CRC10_ATM => "CRC-10/ATM [CRC-10, CRC-10/I-610]",
HashType.CRC10_CDMA2000 => "CRC-10/CDMA2000",
HashType.CRC10_GSM => "CRC-10/GSM",
HashType.CRC11_FLEXRAY => "CRC-11/FLEXRAY [CRC-11]",
HashType.CRC11_UMTS => "CRC-11/UMTS",
HashType.CRC12_CDMA2000 => "CRC-12/CDMA2000",
HashType.CRC12_DECT => "CRC-12/DECT [X-CRC-12]",
HashType.CRC12_GSM => "CRC-12/GSM",
HashType.CRC12_UMTS => "CRC-12/UMTS [CRC-12/3GPP]",
HashType.CRC13_BBC => "CRC-13/BBC",
HashType.CRC14_DARC => "CRC-14/DARC",
HashType.CRC14_GSM => "CRC-14/GSM",
HashType.CRC15_CAN => "CRC-15/CAN [CRC-15]",
HashType.CRC15_MPT1327 => "CRC-15/MPT1327",
HashType.CRC16 => "CRC-16",
HashType.CRC16_ARC => "CRC-16/ARC [ARC, CRC-16, CRC-16/LHA, CRC-IBM]",
HashType.CRC16_CDMA2000 => "CRC-16/CDMA2000",
HashType.CRC16_CMS => "CRC-16/CMS",
HashType.CRC16_DDS110 => "CRC-16/DDS-110",
HashType.CRC16_DECTR => "CRC-16/DECT-R [R-CRC-16]",
HashType.CRC16_DECTX => "CRC-16/DECT-X [X-CRC-16]",
HashType.CRC16_DNP => "CRC-16/DNP",
HashType.CRC16_EN13757 => "CRC-16/EN-13757",
HashType.CRC16_GENIBUS => "CRC-16/GENIBUS [CRC-16/DARC, CRC-16/EPC, CRC-16/EPC-C1G2, CRC-16/I-CODE]",
HashType.CRC16_GSM => "CRC-16/GSM",
HashType.CRC16_IBM3740 => "CRC-16/IBM-3740 [CRC-16/AUTOSAR, CRC-16/CCITT-FALSE]",
HashType.CRC16_IBMSDLC => "CRC-16/IBM-SDLC [CRC-16/ISO-HDLC, CRC-16/ISO-IEC-14443-3-B, CRC-16/X-25, CRC-B, X-25]",
HashType.CRC16_ISOIEC144433A => "CRC-16/ISO-IEC-14443-3-A [CRC-A]",
HashType.CRC16_KERMIT => "CRC-16/KERMIT [CRC-16/BLUETOOTH, CRC-16/CCITT, CRC-16/CCITT-TRUE, CRC-16/V-41-LSB, CRC-CCITT, KERMIT]",
HashType.CRC16_LJ1200 => "CRC-16/LJ1200",
HashType.CRC16_M17 => "CRC-16/M17",
HashType.CRC16_MAXIMDOW => "CRC-16/MAXIM-DOW [CRC-16/MAXIM]",
HashType.CRC16_MCRF4XX => "CRC-16/MCRF4XX",
HashType.CRC16_MODBUS => "CRC-16/MODBUS [MODBUS]",
HashType.CRC16_NRSC5 => "CRC-16/NRSC-5",
HashType.CRC16_OPENSAFETYA => "CRC-16/OPENSAFETY-A",
HashType.CRC16_OPENSAFETYB => "CRC-16/OPENSAFETY-B",
HashType.CRC16_PROFIBUS => "CRC-16/PROFIBUS [CRC-16/IEC-61158-2]",
HashType.CRC16_RIELLO => "CRC-16/RIELLO",
HashType.CRC16_SPIFUJITSU => "CRC-16/SPI-FUJITSU [CRC-16/AUG-CCITT]",
HashType.CRC16_T10DIF => "CRC-16/T10-DIF",
HashType.CRC16_TELEDISK => "CRC-16/TELEDISK",
HashType.CRC16_TMS37157 => "CRC-16/TMS37157",
HashType.CRC16_UMTS => "CRC-16/UMTS [CRC-16/BUYPASS, CRC-16/VERIFONE]",
HashType.CRC16_USB => "CRC-16/USB",
HashType.CRC16_XMODEM => "CRC-16/XMODEM [CRC-16/ACORN, CRC-16/LTE, CRC-16/V-41-MSB, XMODEM, ZMODEM]",
HashType.CRC17_CANFD => "CRC-17/CAN-FD",
HashType.CRC21_CANFD => "CRC-21/CAN-FD",
HashType.CRC24_BLE => "CRC-24/BLE",
HashType.CRC24_FLEXRAYA => "CRC-24/FLEXRAY-A",
HashType.CRC24_FLEXRAYB => "CRC-24/FLEXRAY-B",
HashType.CRC24_INTERLAKEN => "CRC-24/INTERLAKEN",
HashType.CRC24_LTEA => "CRC-24/LTE-A",
HashType.CRC24_LTEB => "CRC-24/LTE-B",
HashType.CRC24_OPENPGP => "CRC-24/OPENPGP",
HashType.CRC24_OS9 => "CRC-24/OS-9",
HashType.CRC30_CDMA => "CRC-30/CDMA",
HashType.CRC31_PHILIPS => "CRC-31/PHILIPS",
HashType.CRC32 => "CRC-32",
HashType.CRC32_AIXM => "CRC-32/AIXM",
HashType.CRC32_AUTOSAR => "CRC-32/AUTOSAR",
HashType.CRC32_BASE91D => "CRC-32/BASE91-D",
HashType.CRC32_BZIP2 => "BZIP2",
HashType.CRC32_CDROMEDC => "CRC-32/CD-ROM-EDC",
HashType.CRC32_CKSUM => "CRC-32/CKSUM",
HashType.CRC32_ISCSI => "CRC-32/ISCSI",
HashType.CRC32_ISOHDLC => "CRC-32/ISO-HDLC",
HashType.CRC32_JAMCRC => "CRC-32/JAMCRC",
HashType.CRC32_MEF => "CRC-32/MEF",
HashType.CRC32_MPEG2 => "CRC-32/MPEG-2",
HashType.CRC32_XFER => "CRC-32/XFER",
HashType.CRC40_GSM => "CRC-40/GSM",
HashType.CRC64 => "CRC-64",
HashType.CRC64_ECMA182 => "CRC-64/ECMA-182, Microsoft implementation",
HashType.CRC64_GOISO => "CRC-64/GO-ISO",
HashType.CRC64_MS => "CRC-64/MS",
HashType.CRC64_NVME => "CRC-64/NVME",
HashType.CRC64_REDIS => "CRC-64/REDIS",
HashType.CRC64_WE => "CRC-64/WE",
HashType.CRC64_XZ => "CRC-64/XZ",
HashType.Fletcher16 => "John G. Fletcher's 16-bit checksum",
HashType.Fletcher32 => "John G. Fletcher's 32-bit checksum",
HashType.Fletcher64 => "John G. Fletcher's 64-bit checksum",
HashType.FNV0_32 => "FNV hash (Variant 0, 32-bit)",
HashType.FNV0_64 => "FNV hash (Variant 0, 64-bit)",
HashType.FNV1_32 => "FNV hash (Variant 1, 32-bit)",
HashType.FNV1_64 => "FNV hash (Variant 1, 64-bit)",
HashType.FNV1a_32 => "FNV hash (Variant 1a, 32-bit)",
HashType.FNV1a_64 => "FNV hash (Variant 1a, 64-bit)",
HashType.MekaCrc => "Custom MEKA checksum",
HashType.MD2 => "MD2 message-digest algorithm",
HashType.MD4 => "MD4 message-digest algorithm",
HashType.MD5 => "MD5 message-digest algorithm",
HashType.RIPEMD128 => "RIPEMD-128 hash",
HashType.RIPEMD160 => "RIPEMD-160 hash",
HashType.RIPEMD256 => "RIPEMD-256 hash",
HashType.RIPEMD320 => "RIPEMD-320 hash",
HashType.SHA1 => "SHA-1 hash",
HashType.SHA256 => "SHA-256 hash",
HashType.SHA384 => "SHA-384 hash",
HashType.SHA512 => "SHA-512 hash",
#if NET8_0_OR_GREATER
HashType.SHA3_256 => "SHA3-256 hash",
HashType.SHA3_384 => "SHA3-384 hash",
HashType.SHA3_512 => "SHA3-512 hash",
HashType.SHAKE128 => "SHAKE128 SHA-3 family hash (256-bit)",
HashType.SHAKE256 => "SHAKE256 SHA-3 family hash (512-bit)",
#endif
HashType.SpamSum => "spamsum fuzzy hash",
HashType.Tiger128_3 => "Tiger 128-bit hash, 3 passes",
HashType.Tiger128_4 => "Tiger 128-bit hash, 4 passes",
HashType.Tiger160_3 => "Tiger 160-bit hash, 3 passes",
HashType.Tiger160_4 => "Tiger 160-bit hash, 4 passes",
HashType.Tiger192_3 => "Tiger 192-bit hash, 3 passes",
HashType.Tiger192_4 => "Tiger 192-bit hash, 4 passes",
HashType.Tiger2_128_3 => "Tiger2 128-bit hash, 3 passes",
HashType.Tiger2_128_4 => "Tiger2 128-bit hash, 4 passes",
HashType.Tiger2_160_3 => "Tiger2 160-bit hash, 3 passes",
HashType.Tiger2_160_4 => "Tiger2 160-bit hash, 4 passes",
HashType.Tiger2_192_3 => "Tiger2 192-bit hash, 3 passes",
HashType.Tiger2_192_4 => "Tiger2 192-bit hash, 4 passes",
HashType.XxHash32 => "xxHash32 hash",
HashType.XxHash64 => "xxHash64 hash",
#if NET462_OR_GREATER || NETCOREAPP
HashType.XxHash3 => "XXH3 64-bit hash",
HashType.XxHash128 => "XXH128 128-bit hash",
#endif
_ => $"{hashType}",
};
}
/// <summary>
/// Get the hash type associated to a string, if possible
/// </summary>
/// TODO: This should be automated instead of hardcoded
public static HashType? GetHashType(this string? str)
{
// Ignore invalid strings
if (string.IsNullOrEmpty(str))
return null;
// Normalize the string before matching
str = str!.Replace("-", string.Empty);
str = str.Replace(" ", string.Empty);
str = str.Replace("/", "_");
str = str.Replace("\\", "_");
str = str.ToLowerInvariant();
// Match based on potential names
return str switch
{
"adler" or "adler32" => HashType.Adler32,
#if NET7_0_OR_GREATER
"blake3" => HashType.BLAKE3,
#endif
"crc1_0" or "crc1_zero" => HashType.CRC1_ZERO,
"crc1_1" or "crc1_one" => HashType.CRC1_ONE,
"crc3_gsm" => HashType.CRC3_GSM,
"crc3_rohc" => HashType.CRC3_ROHC,
"crc4_g704" or "crc4_itu" => HashType.CRC4_G704,
"crc4_interlaken" => HashType.CRC4_INTERLAKEN,
"crc5_epc" or "crc5_epcc1g2" => HashType.CRC5_EPCC1G2,
"crc5_g704" or "crc5_itu" => HashType.CRC5_G704,
"crc5_usb" => HashType.CRC5_USB,
"crc6_cdma2000a" => HashType.CRC6_CDMA2000A,
"crc6_cdma2000b" => HashType.CRC6_CDMA2000B,
"crc6_darc" => HashType.CRC6_DARC,
"crc6_g704" or "crc6_itu" => HashType.CRC6_G704,
"crc6_gsm" => HashType.CRC6_GSM,
"crc7" or "crc7_mmc" => HashType.CRC7_MMC,
"crc7_rohc" => HashType.CRC7_ROHC,
"crc7_umts" => HashType.CRC7_UMTS,
"crc8" => HashType.CRC8,
"crc8_autosar" => HashType.CRC8_AUTOSAR,
"crc8_bluetooth" => HashType.CRC8_BLUETOOTH,
"crc8_cdma2000" => HashType.CRC8_CDMA2000,
"crc8_darc" => HashType.CRC8_DARC,
"crc8_dvbs2" => HashType.CRC8_DVBS2,
"crc8_gsma" => HashType.CRC8_GSMA,
"crc8_gsmb" => HashType.CRC8_GSMB,
"crc8_hitag" => HashType.CRC8_HITAG,
"crc8_i4321" or "crc8_itu" => HashType.CRC8_I4321,
"crc8_icode" => HashType.CRC8_ICODE,
"crc8_lte" => HashType.CRC8_LTE,
"crc8_maximdow" or "crc8_maxim" or "dowcrc" => HashType.CRC8_MAXIMDOW,
"crc8_mifaremad" => HashType.CRC8_MIFAREMAD,
"crc8_nrsc5" => HashType.CRC8_NRSC5,
"crc8_opensafety" => HashType.CRC8_OPENSAFETY,
"crc8_rohc" => HashType.CRC8_ROHC,
"crc8_saej1850" => HashType.CRC8_SAEJ1850,
"crc8_smbus" => HashType.CRC8_SMBUS,
"crc8_tech3250" or "crc8_aes" or "crc8_ebu" => HashType.CRC8_TECH3250,
"crc8_wcdma" => HashType.CRC8_WCDMA,
"crc10_atm" or "crc10" or "crc10_i610" => HashType.CRC10_ATM,
"crc10_cdma2000" => HashType.CRC10_CDMA2000,
"crc10_gsm" => HashType.CRC10_GSM,
"crc11_flexray" or "crc11" => HashType.CRC11_FLEXRAY,
"crc11_umts" => HashType.CRC11_UMTS,
"crc12_cdma2000" => HashType.CRC12_CDMA2000,
"crc12_dect" or "xcrc12" => HashType.CRC12_DECT,
"crc12_gsm" => HashType.CRC12_GSM,
"crc12_umts" or "crc12_3gpp" => HashType.CRC12_UMTS,
"crc13_bbc" => HashType.CRC13_BBC,
"crc14_darc" => HashType.CRC14_DARC,
"crc14_gsm" => HashType.CRC14_GSM,
"crc15_can" or "crc15" => HashType.CRC15_CAN,
"crc15_mpt1327" => HashType.CRC15_MPT1327,
"crc16" => HashType.CRC16,
"crc16_arc" or "arc" or "crc16_lha" or "crcibm" => HashType.CRC16_ARC,
"crc16_cdma2000" => HashType.CRC16_CDMA2000,
"crc16_cms" => HashType.CRC16_CMS,
"crc16_dds110" => HashType.CRC16_DDS110,
"crc16_dectr" or "rcrc16" => HashType.CRC16_DECTR,
"crc16_dectx" or "xcrc16" => HashType.CRC16_DECTX,
"crc16_dnp" => HashType.CRC16_DNP,
"crc16_en13757" => HashType.CRC16_EN13757,
"crc16_genibus" or "crc16_darc" or "crc16_epc" or "crc16_epcc1g2" or "crc16_icode" => HashType.CRC16_GENIBUS,
"crc16_gsm" => HashType.CRC16_GSM,
"crc16_ibm3740" or "crc16_autosar" or "crc16_cittfalse" => HashType.CRC16_IBM3740,
"crc16_ibmsdlc" or "crc16_isohdlc" or "crc16_isoiec144433b" or "crc16_x25" or "crcb" or "x25" => HashType.CRC16_IBMSDLC,
"crc16_isoiec144433a" or "crca" => HashType.CRC16_ISOIEC144433A,
"crc16_kermit" or "crc16_bluetooth" or "crc16_ccitt" or "crc16_ccitttrue" or "crc16_v41lsb" or "crcccitt" or "kermit" => HashType.CRC16_KERMIT,
"crc16_lj1200" => HashType.CRC16_LJ1200,
"crc16_m17" => HashType.CRC16_M17,
"crc16_maximdow" or "crc16_maxim" => HashType.CRC16_MAXIMDOW,
"crc16_mcrf4xx" => HashType.CRC16_MCRF4XX,
"crc16_modbus" or "modbus" => HashType.CRC16_MODBUS,
"crc16_nrsc5" => HashType.CRC16_NRSC5,
"crc16_opensafetya" => HashType.CRC16_OPENSAFETYA,
"crc16_opensafetyb" => HashType.CRC16_OPENSAFETYB,
"crc16_profibus" or "crc16_iec611582" => HashType.CRC16_PROFIBUS,
"crc16_riello" => HashType.CRC16_RIELLO,
"crc16_spifujitsu" or "crc16_augccitt" => HashType.CRC16_SPIFUJITSU,
"crc16_t10dif" => HashType.CRC16_T10DIF,
"crc16_teledisk" => HashType.CRC16_TELEDISK,
"crc16_tms37157" => HashType.CRC16_TMS37157,
"crc16_umts" or "crc16_buypass" or "crc16_verifone" => HashType.CRC16_UMTS,
"crc16_usb" => HashType.CRC16_USB,
"crc16_xmodem" or "crc16_acorn" or "crc16_lte" or "crc16_v41msb" or "xmodem" or "zmodem" => HashType.CRC16_XMODEM,
"crc17_canfd" => HashType.CRC17_CANFD,
"crc21_canfd" => HashType.CRC21_CANFD,
"crc24_ble" => HashType.CRC24_BLE,
"crc24_flexraya" => HashType.CRC24_FLEXRAYA,
"crc24_flexrayb" => HashType.CRC24_FLEXRAYB,
"crc24_interlaken" => HashType.CRC24_INTERLAKEN,
"crc24_ltea" => HashType.CRC24_LTEA,
"crc24_lteb" => HashType.CRC24_LTEB,
"crc24_openpgp" => HashType.CRC24_OPENPGP,
"crc24_os9" => HashType.CRC24_OS9,
"crc30_cdma" => HashType.CRC30_CDMA,
"crc31_philips" => HashType.CRC31_PHILIPS,
"crc32" => HashType.CRC32,
"crc32_aixm" => HashType.CRC32_AIXM,
"crc32_autosar" => HashType.CRC32_AUTOSAR,
"crc32_base91d" => HashType.CRC32_BASE91D,
"crc32_bzip2" => HashType.CRC32_BZIP2,
"crc32_cdromedc" => HashType.CRC32_CDROMEDC,
"crc32_cksum" => HashType.CRC32_CKSUM,
"crc32_iscsi" => HashType.CRC32_ISCSI,
"crc32_isohdlc" => HashType.CRC32_ISOHDLC,
"crc32_jamcrc" => HashType.CRC32_JAMCRC,
"crc32_mef" => HashType.CRC32_MEF,
"crc32_mpeg2" => HashType.CRC32_MPEG2,
"crc32_xfer" => HashType.CRC32_XFER,
"crc40_gsm" => HashType.CRC40_GSM,
"crc64" => HashType.CRC64,
"crc64_ecma182" => HashType.CRC64_ECMA182,
"crc64_goiso" => HashType.CRC64_GOISO,
"crc64_ms" => HashType.CRC64_MS,
"crc64_nvme" => HashType.CRC64_NVME,
"crc64_redis" => HashType.CRC64_REDIS,
"crc64_we" => HashType.CRC64_WE,
"crc64_xz" => HashType.CRC64_XZ,
"fletcher16" => HashType.Fletcher16,
"fletcher32" => HashType.Fletcher32,
"fletcher64" => HashType.Fletcher64,
"fnv0_32" => HashType.FNV0_32,
"fnv0_64" => HashType.FNV0_64,
"fnv1_32" => HashType.FNV1_32,
"fnv1_64" => HashType.FNV1_64,
"fnv1a_32" => HashType.FNV1a_32,
"fnv1a_64" => HashType.FNV1a_64,
"meka" or "mekacrc" or "meka_crc" => HashType.MekaCrc,
"md2" => HashType.MD2,
"md4" => HashType.MD4,
"md5" => HashType.MD5,
"ripemd128" => HashType.RIPEMD128,
"ripemd160" => HashType.RIPEMD160,
"ripemd256" => HashType.RIPEMD256,
"ripemd320" => HashType.RIPEMD320,
"sha1" => HashType.SHA1,
"sha256" => HashType.SHA256,
"sha384" => HashType.SHA384,
"sha512" => HashType.SHA512,
#if NET8_0_OR_GREATER
"sha3_256" => HashType.SHA3_256,
"sha3_384" => HashType.SHA3_384,
"sha3_512" => HashType.SHA3_512,
"shake128" => HashType.SHAKE128,
"shake256" => HashType.SHAKE256,
#endif
"spamsum" => HashType.SpamSum,
"tiger128_3" => HashType.Tiger128_3,
"tiger128_4" => HashType.Tiger128_4,
"tiger160_3" => HashType.Tiger160_3,
"tiger160_4" => HashType.Tiger160_4,
"tiger192_3" => HashType.Tiger192_3,
"tiger192_4" => HashType.Tiger192_4,
"tiger2_128_3" => HashType.Tiger2_128_3,
"tiger2_128_4" => HashType.Tiger2_128_4,
"tiger2_160_3" => HashType.Tiger2_160_3,
"tiger2_160_4" => HashType.Tiger2_160_4,
"tiger2_192_3" => HashType.Tiger2_192_3,
"tiger2_192_4" => HashType.Tiger2_192_4,
"xxh" or "xxh32" or "xxh_32" or "xxhash" or "xxhash32" or "xxhash_32" => HashType.XxHash32,
"xxh64" or "xxh_64" or "xxhash64" or "xxhash_64" => HashType.XxHash64,
#if NET462_OR_GREATER || NETCOREAPP
"xxh3" or "xxh3_64" or "xxhash3" or "xxhash_3" => HashType.XxHash3,
"xxh128" or "xxh_128" or "xxhash128" or "xxhash_128" => HashType.XxHash128,
#endif
_ => null,
};
}
}
}

View File

@@ -178,4 +178,4 @@ namespace SabreTools.Hashing
#endregion
}
}
}

View File

@@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
#if NET40_OR_GREATER || NETCOREAPP
#if NET40_OR_GREATER || NETCOREAPP || NETSTANDARD2_0_OR_GREATER
using System.Threading.Tasks;
#endif
@@ -753,4 +753,4 @@ namespace SabreTools.Hashing
#endregion
}
}
}

View File

@@ -756,6 +756,11 @@ namespace SabreTools.Hashing
#endregion
/// <summary>
/// Custom checksum used by MEKA
/// </summary>
MekaCrc,
#region Message Digest
/// <summary>
@@ -947,4 +952,4 @@ namespace SabreTools.Hashing
#endregion
}
}
}

View File

@@ -237,12 +237,14 @@ namespace SabreTools.Hashing
HashType.FNV1a_32 => new FNV1a_32(),
HashType.FNV1a_64 => new FNV1a_64(),
HashType.MekaCrc => new MekaCrc(),
HashType.MD2 => new MD2(),
HashType.MD4 => new MD4(),
HashType.MD5 => MD5.Create(),
HashType.RIPEMD128 => new RipeMD128(),
HashType.RIPEMD160 => new CryptographicHash.RipeMD160(),
HashType.RIPEMD160 => new RipeMD160(),
HashType.RIPEMD256 => new RipeMD256(),
HashType.RIPEMD320 => new RipeMD320(),
@@ -373,4 +375,4 @@ namespace SabreTools.Hashing
#endregion
}
}
}

View File

@@ -40,4 +40,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
#endregion
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -23,4 +23,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
}
}
}
}
}

View File

@@ -57,4 +57,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
return hashArr;
}
}
}
}

View File

@@ -43,4 +43,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
return hashArr;
}
}
}
}

View File

@@ -147,7 +147,7 @@ namespace SabreTools.Hashing.NonCryptographicHash
private static uint Round(uint acc, uint input)
{
acc += input * XXH_PRIME32_2;
acc = RotateLeft32(acc, 13);
acc = RotateLeft32(acc, 13);
acc *= XXH_PRIME32_1;
return acc;
}
@@ -201,4 +201,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
return Avalanche(hash);
}
}
}
}

View File

@@ -43,4 +43,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
return hashArr;
}
}
}
}

View File

@@ -144,7 +144,7 @@ namespace SabreTools.Hashing.NonCryptographicHash
private static ulong Round(ulong acc, ulong input)
{
acc += unchecked(input * XXH_PRIME64_2);
acc = RotateLeft64(acc, 31);
acc = RotateLeft64(acc, 31);
acc *= XXH_PRIME64_1;
return acc;
}
@@ -199,7 +199,7 @@ namespace SabreTools.Hashing.NonCryptographicHash
return Avalanche(hash);
}
/// <summary>
/// Mixes all bits to finalize the hash.
///
@@ -216,4 +216,4 @@ namespace SabreTools.Hashing.NonCryptographicHash
return hash;
}
}
}
}

View File

@@ -1,8 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<!-- Assembly Properties -->
<TargetFrameworks>net20;net35;net40;net452;net462;net472;net48;netcoreapp3.1;net5.0;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
<TargetFrameworks>net20;net35;net40;net452;net462;net472;net48;netcoreapp3.1;net5.0;net6.0;net7.0;net8.0;net9.0;netstandard2.0;netstandard2.1</TargetFrameworks>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
<IncludeSymbols>true</IncludeSymbols>
@@ -11,7 +11,7 @@
<SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<Version>1.4.2</Version>
<Version>1.5.1</Version>
<!-- Package Properties -->
<Authors>Matt Nadareski</Authors>
@@ -28,12 +28,10 @@
<None Include="../README.md" Pack="true" PackagePath="" />
</ItemGroup>
<!-- Support for old .NET versions -->
<ItemGroup Condition="!$(TargetFramework.StartsWith(`net2`)) AND !$(TargetFramework.StartsWith(`net3`)) AND !$(TargetFramework.StartsWith(`net40`)) AND !$(TargetFramework.StartsWith(`net45`))">
<PackageReference Include="System.IO.Hashing" Version="8.0.0" />
</ItemGroup>
<ItemGroup Condition="$(TargetFramework.StartsWith(`net7`)) OR $(TargetFramework.StartsWith(`net8`)) OR $(TargetFramework.StartsWith(`net9`))">
<PackageReference Include="Blake3" Version="1.1.0" />
<ItemGroup>
<PackageReference Include="Blake3" Version="1.1.0" Condition="$(TargetFramework.StartsWith(`net7`))" />
<PackageReference Include="Blake3" Version="2.0.0" Condition="$(TargetFramework.StartsWith(`net8`)) OR $(TargetFramework.StartsWith(`net9`))" />
<PackageReference Include="System.IO.Hashing" Version="9.0.7" Condition="!$(TargetFramework.StartsWith(`net2`)) AND !$(TargetFramework.StartsWith(`net3`)) AND !$(TargetFramework.StartsWith(`net40`)) AND !$(TargetFramework.StartsWith(`net45`))" />
</ItemGroup>
</Project>

View File

@@ -42,4 +42,4 @@ namespace SabreTools.Hashing.SpamSum
Digest = new byte[SPAMSUM_LENGTH];
}
}
}
}

View File

@@ -0,0 +1,195 @@
using System;
using System.Text.RegularExpressions;
namespace SabreTools.Hashing.SpamSum;
internal static class Comparisons
{
/// <summary>
/// Regex to reduce any sequences longer than 3
/// </summary>
private static Regex _reduceRegex = new("(.)(?<=\\1\\1\\1\\1)", RegexOptions.Compiled);
/// <summary>
/// Compares how similar two SpamSums are to each other
/// </summary>
/// <param name="first">First hash to compare</param>
/// <param name="second">Second hash to compare</param>
/// <returns>-1 on validity failure, 0 if they're not comparable, score from 0 (least similar) to 100 (most similar) otherwise.</returns>
/// <remarks>Implements ssdeep's fuzzy_compare</remarks>
/// <see href="https://github.com/ssdeep-project/ssdeep/blob/df3b860f8918261b3faeec9c7d2c8a241089e6e6/fuzzy.c#L860"/>
public static int FuzzyCompare(string? first, string? second)
{
// If either input is invalid
if (string.IsNullOrEmpty(first) || string.IsNullOrEmpty(second))
return -1;
// Split the string into 3 parts for processing
var firstSplit = first!.Split(':');
var secondSplit = second!.Split(':');
if (firstSplit.Length != 3 || secondSplit.Length != 3)
return -1;
// If any of the required parts are empty
if (firstSplit[0].Length == 0 || firstSplit[2].Length == 0)
return -1;
if (secondSplit[0].Length == 0 || secondSplit[2].Length == 0)
return -1;
// Ensure only second block data before a comma is used
firstSplit[2] = firstSplit[2].Split(',')[0];
secondSplit[2] = secondSplit[2].Split(',')[0];
// Each SpamSum string starts with its block size before the first semicolon.
if (!uint.TryParse(firstSplit[0], out uint firstBlockSize))
return -1;
if (!uint.TryParse(secondSplit[0], out uint secondBlockSize))
return -1;
// Check if blocksizes don't match. Each spamSum is broken up into two blocks.
// fuzzy_compare allows you to compare if one block in one hash is the same
// size as one block in the other hash, even if the other two are non-matching,
// so that's also checked for.
if (firstBlockSize != secondBlockSize
&& (firstBlockSize > uint.MaxValue / 2 || firstBlockSize * 2 != secondBlockSize)
&& (firstBlockSize % 2 == 1 || firstBlockSize / 2 != secondBlockSize))
{
return 0;
}
// Reduce any sequences longer than 3
// These sequences contain very little info and can be reduced as a result
string firstBlockOne = _reduceRegex.Replace(firstSplit[1], string.Empty);
string firstBlockTwo = _reduceRegex.Replace(firstSplit[2], string.Empty);
string secondBlockOne = _reduceRegex.Replace(secondSplit[1], string.Empty);
string secondBlockTwo = _reduceRegex.Replace(secondSplit[2], string.Empty);
// Return 100 immediately if both spamSums are identical.
if (firstBlockSize == secondBlockSize && firstBlockOne == secondBlockOne && firstBlockTwo == secondBlockTwo)
return 100;
// Choose different scoring combinations depending on block sizes present.
if (firstBlockSize <= uint.MaxValue / 2)
{
if (firstBlockSize == secondBlockSize)
{
uint score1 = ScoreStrings(firstBlockOne, secondBlockOne, firstBlockSize);
uint score2 = ScoreStrings(firstBlockTwo, secondBlockTwo, firstBlockSize * 2);
return (int)Math.Max(score1, score2);
}
else if (firstBlockSize * 2 == secondBlockSize)
{
return (int)ScoreStrings(secondBlockOne, firstBlockTwo, secondBlockSize);
}
else
{
return (int)ScoreStrings(firstBlockOne, secondBlockTwo, firstBlockSize);
}
}
else
{
if (firstBlockSize == secondBlockSize)
return (int)ScoreStrings(firstBlockOne, secondBlockOne, firstBlockSize);
else if (firstBlockSize % 2 == 0 && firstBlockSize / 2 == secondBlockSize)
return (int)ScoreStrings(firstBlockOne, secondBlockTwo, firstBlockSize);
else
return 0;
}
}
/// <summary>
/// Checks whether the two SpamSum strings have a common substring of 7 or more characters (as defined in fuzzy_compare's ROLLING_WINDOW size).
/// </summary>
/// <param name="first">First string to score</param>
/// <param name="second">Second string to score</param>
/// <returns>False if there is no common substring of 7 or more characters, true if there is.</returns>
private static bool HasCommmonSubstring(string first, string second)
{
// If either string is less than 7 characters
if (first.Length < 7 || second.Length < 7)
return false;
for (int i = 0; i < first.Length; i++)
{
for (int j = 0; j < second.Length; j++)
{
int currentIndex = 0;
while ((i + currentIndex) < first.Length && (j + currentIndex) < second.Length && first[i + currentIndex] == second[j + currentIndex])
{
currentIndex++;
}
if (currentIndex >= 7)
return true;
}
}
return false;
}
/// <summary>
/// Compares how similar two SpamSums are to each other. Implements ssdeep's fuzzy_compare.
/// </summary>
/// <param name="first">First string to score</param>
/// <param name="second">Second string to score</param>
/// <param name="blockSize">Current blocksize</param>
/// <returns>-1 on validity failure, 0 if they're not comparable, score from 0 (least similar) to 100 (most similar) otherwise.</returns>
private static uint ScoreStrings(string first, string second, uint blockSize)
{
if (!HasCommmonSubstring(first, second))
return 0;
const uint maxLength = 64;
const uint insertCost = 1;
const uint removeCost = 1;
const uint replaceCost = 2;
var firstTraverse = new uint[maxLength + 1];
var secondTraverse = new uint[maxLength + 1];
for (uint secondIndex = 0; secondIndex <= second.Length; secondIndex++)
{
firstTraverse[secondIndex] = secondIndex * removeCost;
}
for (uint firstIndex = 0; firstIndex < first.Length; firstIndex++)
{
secondTraverse[0] = (firstIndex + 1) * insertCost;
for (uint secondIndex = 0; secondIndex < second.Length; secondIndex++)
{
var costA = firstTraverse[secondIndex + 1] + insertCost;
var costD = secondTraverse[secondIndex] + removeCost;
var costR = firstTraverse[secondIndex] + (first[(int)firstIndex] == second[(int)secondIndex] ? 0 : replaceCost);
secondTraverse[secondIndex + 1] = Math.Min(Math.Min(costA, costD), costR);
}
var tempArray = firstTraverse;
firstTraverse = secondTraverse;
secondTraverse = tempArray;
}
long score = firstTraverse[second.Length];
const int spamSumLength = 64;
const int rollingWindow = 7;
const int minBlocksize = 3;
score = (score * spamSumLength) / (first.Length + second.Length);
// Currently, the score ranges from 0-64 (64 being the length of a spamsum), with 0 being the strongest match
// and 64 being the weakest match.
// Change scale to 0-100
score = (100 * score) / spamSumLength;
// Invert scale so 0 is the weakest possible match and 100 is the strongest
score = 100 - score;
// Compensate for small blocksizes, so match isn't reported as overly strong.
if (blockSize >= (99 + rollingWindow) / rollingWindow * minBlocksize)
return (uint)score;
if (score > blockSize / minBlocksize * Math.Min(first.Length, second.Length))
score = blockSize / minBlocksize * Math.Min(first.Length, second.Length);
return (uint)score;
}
}

View File

@@ -443,4 +443,4 @@ namespace SabreTools.Hashing.SpamSum
#endregion
}
}
}

View File

@@ -63,7 +63,7 @@ namespace SabreTools.Hashing.SpamSum
LastH = BH[obh].H;
}
}
public void TryReduceBlockhash()
{
if (BHStart >= BHEnd)
@@ -88,4 +88,4 @@ namespace SabreTools.Hashing.SpamSum
RollMask = RollMask * 2 + 1;
}
}
}
}

View File

@@ -47,10 +47,10 @@ namespace SabreTools.Hashing.SpamSum
H3 <<= 5;
H3 ^= c;
}
/// <summary>
/// Return the current rolling sum
/// </summary>
public uint RollSum() => H1 + H2 + H3;
}
}
}

View File

@@ -36,6 +36,10 @@ namespace SabreTools.Hashing.SpamSum
_state.BH[0].DIndex = 0;
}
/// <inheritdoc cref="Comparisons.FuzzyCompare(string?, string?)"/>
public static int FuzzyCompare(string? firstHash, string? secondHash)
=> Comparisons.FuzzyCompare(firstHash, secondHash);
/// <inheritdoc/>
protected override void HashCore(byte[] array, int ibStart, int cbSize)
{
@@ -353,4 +357,4 @@ namespace SabreTools.Hashing.SpamSum
return n;
}
}
}
}

View File

@@ -67,4 +67,4 @@ namespace SabreTools.Hashing.XxHash
#endregion
}
}
}

View File

@@ -71,4 +71,4 @@ namespace SabreTools.Hashing.XxHash
/// </summary>
XXH_SVE = 6,
}
}
}

View File

@@ -249,4 +249,4 @@ namespace SabreTools.Hashing.XxHash
#endregion
}
}
}

View File

@@ -3,7 +3,7 @@ namespace SabreTools.Hashing.XxHash
internal class XXH3_128Hash
{
public ulong Low { get; set; }
public ulong High { get; set; }
}
}
}

View File

@@ -1,9 +1,9 @@
namespace SabreTools.Hashing.XxHash
{
// Handle unused private fields
#pragma warning disable CS0169
#pragma warning disable CS0414
#pragma warning disable CS0649
#pragma warning disable CS0169
#pragma warning disable CS0414
#pragma warning disable CS0649
/// <summary>
/// Structure for XXH3 streaming API.
@@ -106,7 +106,7 @@ namespace SabreTools.Hashing.XxHash
{
// TODO: XXH3_128bits_reset_withSecret
}
/// <summary>
/// Hash a block of data and append it to the existing hash
/// </summary>
@@ -128,4 +128,4 @@ namespace SabreTools.Hashing.XxHash
return ulong.MaxValue;
}
}
}
}

View File

@@ -1,9 +1,9 @@
namespace SabreTools.Hashing.XxHash
{
// Handle unused private fields
#pragma warning disable CS0169
#pragma warning disable CS0414
#pragma warning disable CS0649
#pragma warning disable CS0169
#pragma warning disable CS0414
#pragma warning disable CS0649
/// <summary>
/// Structure for XXH3 streaming API.
@@ -111,7 +111,7 @@ namespace SabreTools.Hashing.XxHash
{
// TODO: XXH3_64bits_reset_withSecret
}
/// <summary>
/// Hash a block of data and append it to the existing hash
/// </summary>
@@ -133,4 +133,4 @@ namespace SabreTools.Hashing.XxHash
return ulong.MaxValue;
}
}
}
}

View File

@@ -250,6 +250,8 @@ namespace SabreTools.Hashing
{HashType.FNV1a_32, [0x81, 0x1c, 0x9d, 0xc5]},
{HashType.FNV1a_64, [0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25]},
{HashType.MekaCrc, [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]},
{HashType.MD2, [0x83, 0x50, 0xe5, 0xa3, 0xe2, 0x4c, 0x15, 0x3d,
0xf2, 0x27, 0x5c, 0x9f, 0x80, 0x69, 0x27, 0x73]},
{HashType.MD4, [0x31, 0xd6, 0xcf, 0xe0, 0xd1, 0x6a, 0xe9, 0x31,
@@ -550,6 +552,8 @@ namespace SabreTools.Hashing
{HashType.FNV1a_32, "811c9dc5"},
{HashType.FNV1a_64, "cbf29ce484222325"},
{HashType.MekaCrc, "0000000000000000"},
{HashType.MD2, "8350e5a3e24c153df2275c9f80692773"},
{HashType.MD4, "31d6cfe0d16ae931b73c59d7e0c089c0"},
{HashType.MD5, "d41d8cd98f00b204e9800998ecf8427e"},
@@ -620,4 +624,4 @@ namespace SabreTools.Hashing
return _strings[hashType];
}
}
}
}

View File

@@ -1,19 +1,32 @@
#! /bin/bash
#!/bin/bash
# This batch file assumes the following:
# - .NET 9.0 (or newer) SDK is installed and in PATH
# - zip is installed and in PATH
# - Git is installed and in PATH
#
# If any of these are not satisfied, the operation may fail
# in an unpredictable way and result in an incomplete output.
# Optional parameters
USE_ALL=false
INCLUDE_DEBUG=false
NO_BUILD=false
while getopts "b" OPTION
do
NO_ARCHIVE=false
while getopts "udba" OPTION; do
case $OPTION in
u)
USE_ALL=true
;;
d)
INCLUDE_DEBUG=true
;;
b)
NO_BUILD=true
;;
a)
NO_ARCHIVE=true
;;
*)
echo "Invalid option provided"
exit 1
@@ -24,13 +37,115 @@ done
# Set the current directory as a variable
BUILD_FOLDER=$PWD
# Set the current commit hash
COMMIT=$(git log --pretty=%H -1)
# Output the selected options
echo "Selected Options:"
echo " Use all frameworks (-u) $USE_ALL"
echo " Include debug builds (-d) $INCLUDE_DEBUG"
echo " No build (-b) $NO_BUILD"
echo " No archive (-a) $NO_ARCHIVE"
echo " "
# Create the build matrix arrays
FRAMEWORKS=("net9.0")
RUNTIMES=("win-x86" "win-x64" "win-arm64" "linux-x64" "linux-arm64" "osx-x64" "osx-arm64")
# Use expanded lists, if requested
if [ $USE_ALL = true ]; then
FRAMEWORKS=("net20" "net35" "net40" "net452" "net462" "net472" "net48" "netcoreapp3.1" "net5.0" "net6.0" "net7.0" "net8.0" "net9.0")
fi
# Create the filter arrays
SINGLE_FILE_CAPABLE=("net5.0" "net6.0" "net7.0" "net8.0" "net9.0")
VALID_APPLE_FRAMEWORKS=("net6.0" "net7.0" "net8.0" "net9.0")
VALID_CROSS_PLATFORM_FRAMEWORKS=("netcoreapp3.1" "net5.0" "net6.0" "net7.0" "net8.0" "net9.0")
VALID_CROSS_PLATFORM_RUNTIMES=("win-arm64" "linux-x64" "linux-arm64" "osx-x64" "osx-arm64")
# Only build if requested
if [ $NO_BUILD = false ]
then
if [ $NO_BUILD = false ]; then
# Restore Nuget packages for all builds
echo "Restoring Nuget packages"
dotnet restore
# Create Nuget Package
dotnet pack SabreTools.Hashing/SabreTools.Hashing.csproj --output $BUILD_FOLDER
fi
# Build Hasher
for FRAMEWORK in "${FRAMEWORKS[@]}"; do
for RUNTIME in "${RUNTIMES[@]}"; do
# Output the current build
echo "===== Build Hasher - $FRAMEWORK, $RUNTIME ====="
# If we have an invalid combination of framework and runtime
if [[ ! $(echo ${VALID_CROSS_PLATFORM_FRAMEWORKS[@]} | fgrep -w $FRAMEWORK) ]]; then
if [[ $(echo ${VALID_CROSS_PLATFORM_RUNTIMES[@]} | fgrep -w $RUNTIME) ]]; then
echo "Skipped due to invalid combination"
continue
fi
fi
# If we have Apple silicon but an unsupported framework
if [[ ! $(echo ${VALID_APPLE_FRAMEWORKS[@]} | fgrep -w $FRAMEWORK) ]]; then
if [ $RUNTIME = "osx-arm64" ]; then
echo "Skipped due to no Apple Silicon support"
continue
fi
fi
# Only .NET 5 and above can publish to a single file
if [[ $(echo ${SINGLE_FILE_CAPABLE[@]} | fgrep -w $FRAMEWORK) ]]; then
# Only include Debug if set
if [ $INCLUDE_DEBUG = true ]; then
dotnet publish Hasher/Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Debug --self-contained true --version-suffix $COMMIT -p:PublishSingleFile=true
fi
dotnet publish Hasher/Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Release --self-contained true --version-suffix $COMMIT -p:PublishSingleFile=true -p:DebugType=None -p:DebugSymbols=false
else
# Only include Debug if set
if [ $INCLUDE_DEBUG = true ]; then
dotnet publish Hasher/Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Debug --self-contained true --version-suffix $COMMIT
fi
dotnet publish Hasher/Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Release --self-contained true --version-suffix $COMMIT -p:DebugType=None -p:DebugSymbols=false
fi
done
done
fi
# Only create archives if requested
if [ $NO_ARCHIVE = false ]; then
# Create Hasher archives
for FRAMEWORK in "${FRAMEWORKS[@]}"; do
for RUNTIME in "${RUNTIMES[@]}"; do
# Output the current build
echo "===== Archive Hasher - $FRAMEWORK, $RUNTIME ====="
# If we have an invalid combination of framework and runtime
if [[ ! $(echo ${VALID_CROSS_PLATFORM_FRAMEWORKS[@]} | fgrep -w $FRAMEWORK) ]]; then
if [[ $(echo ${VALID_CROSS_PLATFORM_RUNTIMES[@]} | fgrep -w $RUNTIME) ]]; then
echo "Skipped due to invalid combination"
continue
fi
fi
# If we have Apple silicon but an unsupported framework
if [[ ! $(echo ${VALID_APPLE_FRAMEWORKS[@]} | fgrep -w $FRAMEWORK) ]]; then
if [ $RUNTIME = "osx-arm64" ]; then
echo "Skipped due to no Apple Silicon support"
continue
fi
fi
# Only include Debug if set
if [ $INCLUDE_DEBUG = true ]; then
cd $BUILD_FOLDER/Hasher/bin/Debug/${FRAMEWORK}/${RUNTIME}/publish/
zip -r $BUILD_FOLDER/Hasher_${FRAMEWORK}_${RUNTIME}_debug.zip .
fi
cd $BUILD_FOLDER/Hasher/bin/Release/${FRAMEWORK}/${RUNTIME}/publish/
zip -r $BUILD_FOLDER/Hasher_${FRAMEWORK}_${RUNTIME}_release.zip .
done
done
# Reset the directory
cd $BUILD_FOLDER
fi

View File

@@ -6,21 +6,129 @@
# Optional parameters
param(
[Parameter(Mandatory = $false)]
[Alias("UseAll")]
[switch]$USE_ALL,
[Parameter(Mandatory = $false)]
[Alias("IncludeDebug")]
[switch]$INCLUDE_DEBUG,
[Parameter(Mandatory = $false)]
[Alias("NoBuild")]
[switch]$NO_BUILD
[switch]$NO_BUILD,
[Parameter(Mandatory = $false)]
[Alias("NoArchive")]
[switch]$NO_ARCHIVE
)
# Set the current directory as a variable
$BUILD_FOLDER = $PSScriptRoot
# Set the current commit hash
$COMMIT = git log --pretty=format:"%H" -1
# Output the selected options
Write-Host "Selected Options:"
Write-Host " Use all frameworks (-UseAll) $USE_ALL"
Write-Host " Include debug builds (-IncludeDebug) $INCLUDE_DEBUG"
Write-Host " No build (-NoBuild) $NO_BUILD"
Write-Host " No archive (-NoArchive) $NO_ARCHIVE"
Write-Host " "
# Create the build matrix arrays
$FRAMEWORKS = @('net9.0')
$RUNTIMES = @('win-x86', 'win-x64', 'win-arm64', 'linux-x64', 'linux-arm64', 'osx-x64', 'osx-arm64')
# Use expanded lists, if requested
if ($USE_ALL.IsPresent) {
$FRAMEWORKS = @('net20', 'net35', 'net40', 'net452', 'net462', 'net472', 'net48', 'netcoreapp3.1', 'net5.0', 'net6.0', 'net7.0', 'net8.0', 'net9.0')
}
# Create the filter arrays
$SINGLE_FILE_CAPABLE = @('net5.0', 'net6.0', 'net7.0', 'net8.0', 'net9.0')
$VALID_APPLE_FRAMEWORKS = @('net6.0', 'net7.0', 'net8.0', 'net9.0')
$VALID_CROSS_PLATFORM_FRAMEWORKS = @('netcoreapp3.1', 'net5.0', 'net6.0', 'net7.0', 'net8.0', 'net9.0')
$VALID_CROSS_PLATFORM_RUNTIMES = @('win-arm64', 'linux-x64', 'linux-arm64', 'osx-x64', 'osx-arm64')
# Only build if requested
if (!$NO_BUILD.IsPresent)
{
if (!$NO_BUILD.IsPresent) {
# Restore Nuget packages for all builds
Write-Host "Restoring Nuget packages"
dotnet restore
# Create Nuget Package
dotnet pack SabreTools.Hashing\SabreTools.Hashing.csproj --output $BUILD_FOLDER
# Build Hasher
foreach ($FRAMEWORK in $FRAMEWORKS) {
foreach ($RUNTIME in $RUNTIMES) {
# Output the current build
Write-Host "===== Build Hasher - $FRAMEWORK, $RUNTIME ====="
# If we have an invalid combination of framework and runtime
if ($VALID_CROSS_PLATFORM_FRAMEWORKS -notcontains $FRAMEWORK -and $VALID_CROSS_PLATFORM_RUNTIMES -contains $RUNTIME) {
Write-Host "Skipped due to invalid combination"
continue
}
# If we have Apple silicon but an unsupported framework
if ($VALID_APPLE_FRAMEWORKS -notcontains $FRAMEWORK -and $RUNTIME -eq 'osx-arm64') {
Write-Host "Skipped due to no Apple Silicon support"
continue
}
# Only .NET 5 and above can publish to a single file
if ($SINGLE_FILE_CAPABLE -contains $FRAMEWORK) {
# Only include Debug if set
if ($INCLUDE_DEBUG.IsPresent) {
dotnet publish Hasher\Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Debug --self-contained true --version-suffix $COMMIT -p:PublishSingleFile=true
}
dotnet publish Hasher\Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Release --self-contained true --version-suffix $COMMIT -p:PublishSingleFile=true -p:DebugType=None -p:DebugSymbols=false
}
else {
# Only include Debug if set
if ($INCLUDE_DEBUG.IsPresent) {
dotnet publish Hasher\Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Debug --self-contained true --version-suffix $COMMIT
}
dotnet publish Hasher\Hasher.csproj -f $FRAMEWORK -r $RUNTIME -c Release --self-contained true --version-suffix $COMMIT -p:DebugType=None -p:DebugSymbols=false
}
}
}
}
# Only create archives if requested
if (!$NO_ARCHIVE.IsPresent) {
# Create Hasher archives
foreach ($FRAMEWORK in $FRAMEWORKS) {
foreach ($RUNTIME in $RUNTIMES) {
# Output the current build
Write-Host "===== Archive Hasher - $FRAMEWORK, $RUNTIME ====="
# If we have an invalid combination of framework and runtime
if ($VALID_CROSS_PLATFORM_FRAMEWORKS -notcontains $FRAMEWORK -and $VALID_CROSS_PLATFORM_RUNTIMES -contains $RUNTIME) {
Write-Host "Skipped due to invalid combination"
continue
}
# If we have Apple silicon but an unsupported framework
if ($VALID_APPLE_FRAMEWORKS -notcontains $FRAMEWORK -and $RUNTIME -eq 'osx-arm64') {
Write-Host "Skipped due to no Apple Silicon support"
continue
}
# Only include Debug if set
if ($INCLUDE_DEBUG.IsPresent) {
Set-Location -Path $BUILD_FOLDER\Hasher\bin\Debug\${FRAMEWORK}\${RUNTIME}\publish\
7z a -tzip $BUILD_FOLDER\Hasher_${FRAMEWORK}_${RUNTIME}_debug.zip *
}
Set-Location -Path $BUILD_FOLDER\Hasher\bin\Release\${FRAMEWORK}\${RUNTIME}\publish\
7z a -tzip $BUILD_FOLDER\Hasher_${FRAMEWORK}_${RUNTIME}_release.zip *
}
}
# Reset the directory
Set-Location -Path $PSScriptRoot
}