claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 11:14:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

DedupeType influences sort more than anything

Browse Source
This commit is contained in:
Matt Nadareski
2025-01-21 11:07:39 -05:00
parent e17809d104
commit e62c14d1b7
8 changed files with 41 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
SabreTools.DatFiles/DatFile.cs
View File

@@ -335,10 +335,10 @@ namespace SabreTools.DatFiles
/// Perform deduplication based on the deduplication type provided
/// </summary>
/// <param name="dedupeType">Dedupe type that should be used</param>
public void Deduplicate(DedupeType dedupeType)
public void Deduplicate()
{
Items.Deduplicate(dedupeType);
ItemsDB.Deduplicate(dedupeType);
Items.Deduplicate();
ItemsDB.Deduplicate();
}
/// <summary>

10
SabreTools.DatFiles/DatFileTool.cs
View File

@@ -462,7 +462,7 @@ namespace SabreTools.DatFiles
{
// For comparison's sake, we want to use CRC as the base bucketing
datFile.BucketBy(ItemKey.CRC);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
intDat.BucketBy(ItemKey.CRC);
// Then we do a hashwise comparison against the base DAT
@@ -511,7 +511,7 @@ namespace SabreTools.DatFiles
{
// For comparison's sake, we want to use Machine Name as the base bucketing
datFile.BucketBy(ItemKey.Machine);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
intDat.BucketBy(ItemKey.Machine);
// Then we do a namewise comparison against the base DAT
@@ -579,7 +579,7 @@ namespace SabreTools.DatFiles
{
// For comparison's sake, we want to use CRC as the base bucketing
datFile.BucketBy(ItemKey.CRC);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
intDat.BucketBy(ItemKey.CRC);
// Then we do a hashwise comparison against the base DAT
@@ -621,7 +621,7 @@ namespace SabreTools.DatFiles
{
// For comparison's sake, we want to use Machine Name as the base bucketing
datFile.BucketBy(ItemKey.Machine);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
intDat.BucketBy(ItemKey.Machine);
// Then we do a namewise comparison against the base DAT
@@ -680,7 +680,7 @@ namespace SabreTools.DatFiles
else
{
intDat.BucketBy(ItemKey.CRC);
intDat.Deduplicate(DedupeType.Full);
intDat.Deduplicate();
}
// Then we compare against the base DAT

23
SabreTools.DatFiles/Enums.cs
View File

@@ -169,29 +169,6 @@ namespace SabreTools.DatFiles
ALL = ulong.MaxValue,
}
/// <summary>
/// Determines the DAT deduplication type
/// </summary>
public enum DedupeType
{
/// <summary>
/// No deduplication
/// </summary>
None = 0,
/// <summary>
/// Deduplicate across all available fields
/// </summary>
/// <remarks>Requires sorting by any hash</remarks>
Full,
/// <summary>
/// Deduplicate on a per-machine basis
/// </summary>
/// <remarks>Requires sorting by machine</remarks>
Game,
}
/// <summary>
/// Determines merging tag handling for DAT output
/// </summary>

24
SabreTools.DatFiles/ItemDictionary.cs
View File

@@ -30,11 +30,6 @@ namespace SabreTools.DatFiles
/// </summary>
private ItemKey _bucketedBy = ItemKey.NULL;
/// <summary>
/// Determine merging type for all items
/// </summary>
private DedupeType _mergedBy = DedupeType.None;
/// <summary>
/// Internal dictionary for the class
/// </summary>
@@ -381,22 +376,10 @@ namespace SabreTools.DatFiles
}
/// <summary>
/// Perform deduplication based on the deduplication type provided
/// Perform deduplication on the current sorted dictionary
/// </summary>
/// <param name="dedupeType">Dedupe type that should be used</param>
public void Deduplicate(DedupeType dedupeType)
public void Deduplicate()
{
// Set the sorted type
_mergedBy = dedupeType;
// If no deduplication is requested, just return
if (dedupeType == DedupeType.None)
return;
// Ensure Game deduplication is valid
if (dedupeType == DedupeType.Game && _bucketedBy != ItemKey.Machine)
return;
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(SortedKeys, Core.Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
@@ -579,9 +562,6 @@ namespace SabreTools.DatFiles
// Set the sorted type
_bucketedBy = bucketBy;
// Reset the merged type since this might change the merge
_mergedBy = DedupeType.None;
// First do the initial sort of all of the roms inplace
List<string> oldkeys = [.. SortedKeys];

13
SabreTools.DatFiles/ItemDictionaryDB.cs
View File

@@ -679,19 +679,10 @@ namespace SabreTools.DatFiles
}
/// <summary>
/// Perform deduplication based on the deduplication type provided
/// Perform deduplication on the current sorted dictionary
/// </summary>
/// <param name="dedupeType">Dedupe type that should be used</param>
public void Deduplicate(DedupeType dedupeType)
public void Deduplicate()
{
// If no deduplication is requested, just return
if (dedupeType == DedupeType.None)
return;
// Ensure Game deduplication is valid
if (dedupeType == DedupeType.Game && _bucketedBy != ItemKey.Machine)
return;
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(SortedKeys, Core.Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER

4
SabreTools.DatTools/Cleaner.cs
View File

@@ -108,12 +108,12 @@ namespace SabreTools.DatTools
if (DedupeRoms == DedupeType.Full)
{
datFile.BucketBy(ItemKey.CRC);
datFile.Deduplicate(DedupeRoms);
datFile.Deduplicate();
}
else if (DedupeRoms == DedupeType.Game)
{
datFile.BucketBy(ItemKey.Machine);
datFile.Deduplicate(DedupeRoms);
datFile.Deduplicate();
}
// Process description to machine name

23
SabreTools.DatTools/Enums.cs
View File

@@ -1,5 +1,28 @@
namespace SabreTools.DatTools
{
/// <summary>
/// Determines the DAT deduplication type
/// </summary>
public enum DedupeType
{
/// <summary>
/// No deduplication
/// </summary>
None = 0,
/// <summary>
/// Deduplicate across all available fields
/// </summary>
/// <remarks>Requires sorting by any hash</remarks>
Full,
/// <summary>
/// Deduplicate on a per-machine basis
/// </summary>
/// <remarks>Requires sorting by machine</remarks>
Game,
}
/// <summary>
/// Determines which files should be skipped in DFD
/// </summary>

8
SabreTools.DatTools/Verification.cs
View File

@@ -211,12 +211,12 @@ namespace SabreTools.DatTools
if (hashOnly)
{
datFile.BucketBy(ItemKey.CRC);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
}
else
{
datFile.BucketBy(ItemKey.Machine);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
}
// Then mark items for removal
@@ -261,12 +261,12 @@ namespace SabreTools.DatTools
if (hashOnly)
{
datFile.BucketBy(ItemKey.CRC);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
}
else
{
datFile.BucketBy(ItemKey.Machine);
datFile.Deduplicate(DedupeType.Full);
datFile.Deduplicate();
}
// Then mark items for removal