claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

BucketedBy superseded by Field

Browse Source
This commit is contained in:
Matt Nadareski
2020-08-22 23:02:30 -07:00
parent d4be402380
commit dc448a5008
8 changed files with 76 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
RombaSharp/Features/RefreshDats.cs
View File

@@ -63,7 +63,7 @@ contents of any changed dats.";
DatFile datroot = DatFile.Create();
datroot.Header.Type = "SuperDAT";
datroot.PopulateFromDir(_dats, asFiles: TreatAsFiles.CHDs);
datroot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
datroot.Items.BucketBy(Field.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)
List<string> databaseDats = new List<string>();
@@ -93,7 +93,7 @@ contents of any changed dats.";
}
}
datroot.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
datroot.Items.BucketBy(Field.MachineName, DedupeType.None, norename: true);
watch.Stop();

2
RombaSharp/Features/RescanDepots.cs
View File

@@ -65,7 +65,7 @@ namespace RombaSharp.Features
// Now rescan the depot itself
DatFile depot = DatFile.Create();
depot.PopulateFromDir(depotname, asFiles: TreatAsFiles.CHDs);
depot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
depot.Items.BucketBy(Field.SHA1, DedupeType.None);
// Set the base queries to use
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";

56
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -252,8 +252,8 @@ namespace SabreTools.Library.DatFiles
if (updateFields.Intersect(DatItem.DatItemFields).Any())
{
// For comparison's sake, we want to use CRC as the base bucketing
Items.BucketBy(BucketedBy.CRC, DedupeType.Full);
intDat.Items.BucketBy(BucketedBy.CRC, DedupeType.None);
Items.BucketBy(Field.CRC, DedupeType.Full);
intDat.Items.BucketBy(Field.CRC, DedupeType.None);
// Then we do a hashwise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
@@ -282,8 +282,8 @@ namespace SabreTools.Library.DatFiles
if (updateFields.Intersect(DatItem.MachineFields).Any())
{
// For comparison's sake, we want to use Machine Name as the base bucketing
Items.BucketBy(BucketedBy.Game, DedupeType.Full);
intDat.Items.BucketBy(BucketedBy.Game, DedupeType.None);
Items.BucketBy(Field.MachineName, DedupeType.Full);
intDat.Items.BucketBy(Field.MachineName, DedupeType.None);
// Then we do a namewise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
@@ -356,9 +356,9 @@ namespace SabreTools.Library.DatFiles
{
// For comparison's sake, we want to use a base ordering
if (useGames)
Items.BucketBy(BucketedBy.Game, DedupeType.None);
Items.BucketBy(Field.MachineName, DedupeType.None);
else
Items.BucketBy(BucketedBy.CRC, DedupeType.None);
Items.BucketBy(Field.CRC, DedupeType.None);
// Now we want to compare each input DAT against the base
foreach (ParentablePath path in inputs)
@@ -373,9 +373,9 @@ namespace SabreTools.Library.DatFiles
// For comparison's sake, we want to a the base bucketing
if (useGames)
intDat.Items.BucketBy(BucketedBy.Game, DedupeType.None);
intDat.Items.BucketBy(Field.MachineName, DedupeType.None);
else
intDat.Items.BucketBy(BucketedBy.CRC, DedupeType.Full);
intDat.Items.BucketBy(Field.CRC, DedupeType.Full);
// Then we compare against the base DAT
List<string> keys = intDat.Items.Keys.ToList();
@@ -504,7 +504,7 @@ namespace SabreTools.Library.DatFiles
watch.Stop();
// Then, ensure that the internal dat can be bucketed in the best possible way
Items.BucketBy(BucketedBy.CRC, DedupeType.None);
Items.BucketBy(Field.CRC, DedupeType.None);
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating all output DATs");
@@ -939,7 +939,7 @@ namespace SabreTools.Library.DatFiles
try
{
// Bucket by game first
Items.BucketBy(BucketedBy.Game, DedupeType.None);
Items.BucketBy(Field.MachineName, DedupeType.None);
// Create a new set of mappings based on the items
var map = new Dictionary<string, Dictionary<Field, string>>();
@@ -1188,7 +1188,7 @@ namespace SabreTools.Library.DatFiles
public void OneGamePerRegion()
{
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
Items.BucketBy(Field.MachineName, DedupeType.None, norename: true);
// Then we want to get a mapping of all machines to parents
Dictionary<string, List<string>> parents = new Dictionary<string, List<string>>();
@@ -1377,7 +1377,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.User("Creating device non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, mergeroms, norename: true);
Items.BucketBy(Field.MachineName, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
while (AddRomsFromDevices(false, false)) ;
@@ -1396,7 +1396,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.User("Creating fully non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, mergeroms, norename: true);
Items.BucketBy(Field.MachineName, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
while (AddRomsFromDevices(true, true)) ;
@@ -1419,7 +1419,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.User("Creating merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, mergeroms, norename: true);
Items.BucketBy(Field.MachineName, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
AddRomsFromChildren();
@@ -1441,7 +1441,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.User("Creating non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, mergeroms, norename: true);
Items.BucketBy(Field.MachineName, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
AddRomsFromParent();
@@ -1463,7 +1463,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.User("Creating split sets from the DAT");
// For sake of ease, the first thing we want to do is bucket by game
Items.BucketBy(BucketedBy.Game, mergeroms, norename: true);
Items.BucketBy(Field.MachineName, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
RemoveRomsFromChild();
@@ -1949,7 +1949,7 @@ namespace SabreTools.Library.DatFiles
// If the output type isn't set already, get the internal output type
Header.DatFormat = (Header.DatFormat == 0 ? currentPath.GetDatFormat() : Header.DatFormat);
Items.SetBucketedBy(BucketedBy.CRC); // Setting this because it can reduce issues later
Items.SetBucketedBy(Field.CRC); // Setting this because it can reduce issues later
// Now parse the correct type of DAT
try
@@ -2075,7 +2075,7 @@ namespace SabreTools.Library.DatFiles
}
// Get the key and add the file
key = item.GetKey(BucketedBy.CRC);
key = item.GetKey(Field.CRC);
Items.Add(key, item);
return key;
@@ -2247,7 +2247,7 @@ namespace SabreTools.Library.DatFiles
{
// Add the list if it doesn't exist already
Rom rom = new Rom(baseFile);
Items.Add(rom.GetKey(BucketedBy.CRC), rom);
Items.Add(rom.GetKey(Field.CRC), rom);
Globals.Logger.User($"File added: {Path.GetFileNameWithoutExtension(item)}{Environment.NewLine}");
}
else
@@ -2371,7 +2371,7 @@ namespace SabreTools.Library.DatFiles
SetDatItemInfo(datItem, item, parent, basepath);
// Add the file information to the DAT
string key = datItem.GetKey(BucketedBy.CRC);
string key = datItem.GetKey(Field.CRC);
Items.Add(key, datItem);
Globals.Logger.User($"File added: {datItem.Name}{Environment.NewLine}");
@@ -2563,7 +2563,7 @@ namespace SabreTools.Library.DatFiles
return success;
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
Items.BucketBy(Field.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = Items.SortedKeys.ToList();
@@ -3135,7 +3135,7 @@ namespace SabreTools.Library.DatFiles
return success;
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
Items.BucketBy(Field.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = Items.SortedKeys.ToList();
@@ -3239,7 +3239,7 @@ namespace SabreTools.Library.DatFiles
if (hashOnly)
{
// First we need to bucket and dedupe by hash to get duplicates
Items.BucketBy(BucketedBy.CRC, DedupeType.Full);
Items.BucketBy(Field.CRC, DedupeType.Full);
// Then follow the same tactics as before
var keys = Items.SortedKeys.ToList();
@@ -3493,7 +3493,7 @@ namespace SabreTools.Library.DatFiles
public bool SplitByLevel(string outDir, bool shortname, bool basedat)
{
// First, bucket by games so that we can do the right thing
Items.BucketBy(BucketedBy.Game, DedupeType.None, lower: false, norename: true);
Items.BucketBy(Field.MachineName, DedupeType.None, lower: false, norename: true);
// Create a temporary DAT to add things to
DatFile tempDat = Create(Header);
@@ -3754,7 +3754,7 @@ namespace SabreTools.Library.DatFiles
if (Items.RomCount + Items.DiskCount == 0)
Items.RecalculateStats();
Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
Items.BucketBy(Field.MachineName, DedupeType.None, norename: true);
var consoleOutput = BaseReport.Create(StatReportFormat.None, null, true, true);
consoleOutput.ReplaceStatistics(Header.FileName, Items.Keys.Count(), Items);
@@ -3762,12 +3762,12 @@ namespace SabreTools.Library.DatFiles
// Bucket and dedupe according to the flag
if (Header.DedupeRoms == DedupeType.Full)
Items.BucketBy(BucketedBy.CRC, Header.DedupeRoms, norename: norename);
Items.BucketBy(Field.CRC, Header.DedupeRoms, norename: norename);
else if (Header.DedupeRoms == DedupeType.Game)
Items.BucketBy(BucketedBy.Game, Header.DedupeRoms, norename: norename);
Items.BucketBy(Field.MachineName, Header.DedupeRoms, norename: norename);
// Bucket roms by game name, if not already
Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: norename);
Items.BucketBy(Field.MachineName, DedupeType.None, norename: norename);
// Output the number of items we're going to be writing
Globals.Logger.User($"A total of {Items.TotalCount} items will be written out to '{Header.FileName}'");

20
SabreTools.Library/DatFiles/Enums.cs
View File

@@ -2,26 +2,6 @@
namespace SabreTools.Library.DatFiles
{
/// <summary>
/// Determines how the current dictionary is bucketed by
/// </summary>
/// TODO: Can we use "Field" instead of this? How much more stupidly complex would that make things?
public enum BucketedBy
{
Default = 0,
Size,
CRC,
MD5,
#if NET_FRAMEWORK
RIPEMD160,
#endif
SHA1,
SHA256,
SHA384,
SHA512,
Game,
}
/// <summary>
/// DAT output formats
/// </summary>

34
SabreTools.Library/DatFiles/ItemDictionary.cs
View File

@@ -23,7 +23,7 @@ namespace SabreTools.Library.DatFiles
/// <summary>
/// Determine the bucketing key for all items
/// </summary>
private BucketedBy bucketedBy;
private Field bucketedBy;
/// <summary>
/// Determine merging type for all items
@@ -340,10 +340,10 @@ namespace SabreTools.Library.DatFiles
}
/// <summary>
/// Override the internal BucketedBy value
/// Override the internal Field value
/// </summary>
/// <param name="newBucket"></param>
public void SetBucketedBy(BucketedBy newBucket)
public void SetBucketedBy(Field newBucket)
{
bucketedBy = newBucket;
}
@@ -544,7 +544,7 @@ namespace SabreTools.Library.DatFiles
/// </summary>
public ItemDictionary()
{
bucketedBy = BucketedBy.Default;
bucketedBy = Field.NULL;
mergedBy = DedupeType.None;
items = new Dictionary<string, List<DatItem>>();
}
@@ -556,11 +556,11 @@ namespace SabreTools.Library.DatFiles
/// <summary>
/// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method
/// </summary>
/// <param name="bucketBy">BucketedBy enum representing how to bucket the individual items</param>
/// <param name="bucketBy">Field enum representing how to bucket the individual items</param>
/// <param name="dedupeType">Dedupe type that should be used</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
public void BucketBy(BucketedBy bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true)
public void BucketBy(Field bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true)
{
// If we have a situation where there's no dictionary or no keys at all, we skip
if (items == null || items.Count == 0)
@@ -625,7 +625,7 @@ namespace SabreTools.Library.DatFiles
DatItem.Sort(ref sortedlist, false);
// If we're merging the roms, do so
if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == BucketedBy.Game))
if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == Field.MachineName))
sortedlist = DatItem.Merge(sortedlist);
// Add the list back to the dictionary
@@ -778,39 +778,39 @@ namespace SabreTools.Library.DatFiles
}
/// <summary>
/// Get the highest-order BucketedBy value that represents the statistics
/// Get the highest-order Field value that represents the statistics
/// </summary>
private BucketedBy GetBestAvailable()
private Field GetBestAvailable()
{
// If all items are supposed to have a SHA-512, we bucket by that
if (RomCount + DiskCount - NodumpCount == SHA512Count)
return BucketedBy.SHA512;
return Field.SHA512;
// If all items are supposed to have a SHA-384, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA384Count)
return BucketedBy.SHA384;
return Field.SHA384;
// If all items are supposed to have a SHA-256, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA256Count)
return BucketedBy.SHA256;
return Field.SHA256;
// If all items are supposed to have a SHA-1, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA1Count)
return BucketedBy.SHA1;
return Field.SHA1;
#if NET_FRAMEWORK
// If all items are supposed to have a RIPEMD160, we bucket by that
else if (RomCount + DiskCount - NodumpCount == RIPEMD160Count)
return BucketedBy.RIPEMD160;
return Field.RIPEMD160;
#endif
// If all items are supposed to have a MD5, we bucket by that
else if (RomCount + DiskCount - NodumpCount == MD5Count)
return BucketedBy.MD5;
return Field.MD5;
// Otherwise, we bucket by CRC
else
return BucketedBy.CRC;
return Field.CRC;
}
/// <summary>
@@ -1005,7 +1005,7 @@ namespace SabreTools.Library.DatFiles
Globals.Logger.Verbose($"Beginning stat collection for '{file}'", false);
List<string> games = new List<string>();
DatFile datdata = DatFile.CreateAndParse(file.CurrentPath);
datdata.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
datdata.Items.BucketBy(Field.MachineName, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
Globals.Logger.User($"Adding stats for file '{file}'\n", false);

21
SabreTools.Library/DatItems/DatItem.cs
View File

@@ -897,11 +897,12 @@ namespace SabreTools.Library.DatItems
/// <summary>
/// Get the dictionary key that should be used for a given item and bucketing type
/// </summary>
/// <param name="bucketedBy">BucketedBy enum representing what key to get</param>
/// <param name="bucketedBy">Field value representing what key to get</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
/// <returns>String representing the key to be used for the DatItem</returns>
public virtual string GetKey(BucketedBy bucketedBy, bool lower = true, bool norename = true)
/// TODO: What other fields can we reasonably allow bucketing on?
public virtual string GetKey(Field bucketedBy, bool lower = true, bool norename = true)
{
// Set the output key as the default blank string
string key = string.Empty;
@@ -909,11 +910,11 @@ namespace SabreTools.Library.DatItems
// Now determine what the key should be based on the bucketedBy value
switch (bucketedBy)
{
case BucketedBy.CRC:
case Field.CRC:
key = Constants.CRCZero;
break;
case BucketedBy.Game:
case Field.MachineName:
key = (norename ? string.Empty
: Source.Index.ToString().PadLeft(10, '0')
+ "-")
@@ -929,29 +930,29 @@ namespace SabreTools.Library.DatItems
key = WebUtility.HtmlEncode(key);
break;
case BucketedBy.MD5:
case Field.MD5:
key = Constants.MD5Zero;
break;
#if NET_FRAMEWORK
case BucketedBy.RIPEMD160:
case Field.RIPEMD160:
key = Constants.RIPEMD160Zero;
break;
#endif
case BucketedBy.SHA1:
case Field.SHA1:
key = Constants.SHA1Zero;
break;
case BucketedBy.SHA256:
case Field.SHA256:
key = Constants.SHA256Zero;
break;
case BucketedBy.SHA384:
case Field.SHA384:
key = Constants.SHA384Zero;
break;
case BucketedBy.SHA512:
case Field.SHA512:
key = Constants.SHA512Zero;
break;
}

16
SabreTools.Library/DatItems/Disk.cs
View File

@@ -668,11 +668,11 @@ namespace SabreTools.Library.DatItems
/// <summary>
/// Get the dictionary key that should be used for a given item and bucketing type
/// </summary>
/// <param name="bucketedBy">BucketedBy enum representing what key to get</param>
/// <param name="bucketedBy">Field enum representing what key to get</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
/// <returns>String representing the key to be used for the DatItem</returns>
public override string GetKey(BucketedBy bucketedBy, bool lower = true, bool norename = true)
public override string GetKey(Field bucketedBy, bool lower = true, bool norename = true)
{
// Set the output key as the default blank string
string key = string.Empty;
@@ -680,29 +680,29 @@ namespace SabreTools.Library.DatItems
// Now determine what the key should be based on the bucketedBy value
switch (bucketedBy)
{
case BucketedBy.MD5:
case Field.MD5:
key = MD5;
break;
#if NET_FRAMEWORK
case BucketedBy.RIPEMD160:
case Field.RIPEMD160:
key = RIPEMD160;
break;
#endif
case BucketedBy.SHA1:
case Field.SHA1:
key = SHA1;
break;
case BucketedBy.SHA256:
case Field.SHA256:
key = SHA256;
break;
case BucketedBy.SHA384:
case Field.SHA384:
key = SHA384;
break;
case BucketedBy.SHA512:
case Field.SHA512:
key = SHA512;
break;

18
SabreTools.Library/DatItems/Rom.cs
View File

@@ -755,11 +755,11 @@ namespace SabreTools.Library.DatItems
/// <summary>
/// Get the dictionary key that should be used for a given item and bucketing type
/// </summary>
/// <param name="bucketedBy">BucketedBy enum representing what key to get</param>
/// <param name="bucketedBy">Field enum representing what key to get</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
/// <returns>String representing the key to be used for the DatItem</returns>
public override string GetKey(BucketedBy bucketedBy, bool lower = true, bool norename = true)
public override string GetKey(Field bucketedBy, bool lower = true, bool norename = true)
{
// Set the output key as the default blank string
string key = string.Empty;
@@ -767,33 +767,33 @@ namespace SabreTools.Library.DatItems
// Now determine what the key should be based on the bucketedBy value
switch (bucketedBy)
{
case BucketedBy.CRC:
case Field.CRC:
key = CRC;
break;
case BucketedBy.MD5:
case Field.MD5:
key = MD5;
break;
#if NET_FRAMEWORK
case BucketedBy.RIPEMD160:
case Field.RIPEMD160:
key = RIPEMD160;
break;
#endif
case BucketedBy.SHA1:
case Field.SHA1:
key = SHA1;
break;
case BucketedBy.SHA256:
case Field.SHA256:
key = SHA256;
break;
case BucketedBy.SHA384:
case Field.SHA384:
key = SHA384;
break;
case BucketedBy.SHA512:
case Field.SHA512:
key = SHA512;
break;