claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Maintain consistent terminology

Browse Source
This commit is contained in:
Matt Nadareski
2020-07-15 10:47:13 -07:00
parent 4e406604c2
commit 6f875c3460
7 changed files with 112 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
RombaSharp/RombaSharp.Help.cs
View File

@@ -1595,6 +1595,7 @@ contents of any changed dats.";
unneeded.Add(hash);
}
}
datroot.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
watch.Stop();
@@ -1691,6 +1692,7 @@ contents of any changed dats.";
// Now rescan the depot itself
DatFile depot = DatFile.Create();
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
depot.PopulateFromDir(depotname, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null);
depot.BucketBy(BucketedBy.SHA1, DedupeType.None);

155
RombaSharp/RombaSharp.Helpers.cs
View File

@@ -252,97 +252,102 @@ namespace RombaSharp
DatFile tempdat = DatFile.CreateAndParse(fullpath);
// If the Dat wasn't empty, add the information
SqliteCommand slc;
if (tempdat.GetCount() != 0)
SqliteCommand slc = null;
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Loop through the parsed entries
bool hasItems = false;
foreach (string romkey in tempdat.Keys)
{
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Loop through the parsed entries
foreach (string romkey in tempdat.Keys)
foreach (DatItem datItem in tempdat[romkey])
{
foreach (DatItem datItem in tempdat[romkey])
{
Globals.Logger.Verbose($"Checking and adding file '{datItem.Name}'");
Globals.Logger.Verbose($"Checking and adding file '{datItem.Name}'");
if (datItem.ItemType == ItemType.Rom)
if (datItem.ItemType == ItemType.Rom)
{
Rom rom = (Rom)datItem;
hasItems = true;
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcquery += $" (\"{rom.CRC}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5query += $" (\"{rom.MD5}\"),";
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
Rom rom = (Rom)datItem;
sha1query += $" (\"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcquery += $" (\"{rom.CRC}\"),";
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5query += $" (\"{rom.MD5}\"),";
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
else if (datItem.ItemType == ItemType.Disk)
}
else if (datItem.ItemType == ItemType.Disk)
{
Disk disk = (Disk)datItem;
hasItems = true;
if (!string.IsNullOrWhiteSpace(disk.MD5))
md5query += $" (\"{disk.MD5}\"),";
if (!string.IsNullOrWhiteSpace(disk.SHA1))
{
Disk disk = (Disk)datItem;
sha1query += $" (\"{disk.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(disk.MD5))
md5query += $" (\"{disk.MD5}\"),";
if (!string.IsNullOrWhiteSpace(disk.SHA1))
{
sha1query += $" (\"{disk.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(disk.MD5))
md5sha1query += $" (\"{disk.MD5}\", \"{disk.SHA1}\"),";
}
md5sha1query += $" (\"{disk.MD5}\", \"{disk.SHA1}\"),";
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
}
string datquery = $"INSERT OR IGNORE INTO dat (hash) VALUES (\"{dat.SHA1}\")";
slc = new SqliteCommand(datquery, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
// Only add the DAT if it's non-empty
if (hasItems)
{
string datquery = $"INSERT OR IGNORE INTO dat (hash) VALUES (\"{dat.SHA1}\")";
slc = new SqliteCommand(datquery, dbc);
slc.ExecuteNonQuery();
}
slc?.Dispose();
}
#endregion

53
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -35,7 +35,7 @@ namespace SabreTools.Library.DatFiles
internal DatStats DatStats = new DatStats();
/// <summary>
/// Determine the sorting key for all items
/// Determine the bucketing key for all items
/// </summary>
private BucketedBy BucketedBy;
@@ -136,29 +136,13 @@ namespace SabreTools.Library.DatFiles
return false;
}
/// <summary>
/// Get total item count statistic
/// </summary>
public long GetCount()
{
return DatStats.Count;
}
/// <summary>
/// Get the FileName header value
/// </summary>
public string GetFileName()
{
return DatHeader.FileName;
}
/// <summary>
/// Get the keys from the file dictionary
/// </summary>
/// <returns>List of the keys</returns>
public List<string> Keys
{
get { return Items.Keys.Select(item => (string)item.Clone()).ToList(); }
get { return Items.Keys.ToList(); }
}
/// <summary>
@@ -198,19 +182,6 @@ namespace SabreTools.Library.DatFiles
Items[key].Remove(value);
}
/// <summary>
/// Remove a range of values from the file dictionary if they exists
/// </summary>
/// <param name="key">Key in the dictionary to remove from</param>
/// <param name="value">Value to remove from the dictionary</param>
public void RemoveRange(string key, List<DatItem> value)
{
foreach (DatItem item in value)
{
Remove(key, item);
}
}
/// <summary>
/// Set the Date header value
/// </summary>
@@ -260,9 +231,9 @@ namespace SabreTools.Library.DatFiles
#region Bucketing
/// <summary>
/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
/// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method
/// </summary>
/// <param name="bucketBy">BucketedBy enum representing how to sort the individual items</param>
/// <param name="bucketBy">BucketedBy enum representing how to bucket the individual items</param>
/// <param name="dedupeType">Dedupe type that should be used</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
@@ -831,7 +802,7 @@ namespace SabreTools.Library.DatFiles
// If we are matching based on DatItem fields of any sort
if (updateFields.Intersect(datItemFields).Any())
{
// For comparison's sake, we want to use CRC as the base ordering
// For comparison's sake, we want to use CRC as the base bucketing
BucketBy(BucketedBy.CRC, DedupeType.Full);
intDat.BucketBy(BucketedBy.CRC, DedupeType.None);
@@ -1138,7 +1109,7 @@ namespace SabreTools.Library.DatFiles
// If we are matching based on Machine fields of any sort
if (updateFields.Intersect(machineFields).Any())
{
// For comparison's sake, we want to use Machine Name as the base ordering
// For comparison's sake, we want to use Machine Name as the base bucketing
BucketBy(BucketedBy.Game, DedupeType.Full);
intDat.BucketBy(BucketedBy.Game, DedupeType.None);
@@ -1253,7 +1224,7 @@ namespace SabreTools.Library.DatFiles
DatFile intDat = Create();
intDat.Parse(path, 1, keep: true);
// For comparison's sake, we want to use CRC as the base ordering
// For comparison's sake, we want to use CRC as the base bucketing
intDat.BucketBy(BucketedBy.CRC, DedupeType.Full);
// Then we do a hashwise comparison against the base DAT
@@ -1326,7 +1297,7 @@ namespace SabreTools.Library.DatFiles
outDats = outDatsArray.ToList();
watch.Stop();
// Then, ensure that the internal dat can be sorted in the best possible way
// Then, ensure that the internal dat can be bucketed in the best possible way
BucketBy(BucketedBy.CRC, DedupeType.None);
// Now, loop through the dictionary and populate the correct DATs
@@ -2253,7 +2224,7 @@ namespace SabreTools.Library.DatFiles
if (directories.Count == 0)
return success;
// Now that we have a list of depots, we want to sort the input DAT by SHA-1
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
BucketBy(BucketedBy.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
@@ -2827,7 +2798,7 @@ namespace SabreTools.Library.DatFiles
if (directories.Count == 0)
return success;
// Now that we have a list of depots, we want to sort the input DAT by SHA-1
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
BucketBy(BucketedBy.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
@@ -2918,7 +2889,7 @@ namespace SabreTools.Library.DatFiles
// If we are checking hashes only, essentially diff the inputs
if (hashOnly)
{
// First we need to sort and dedupe by hash to get duplicates
// First we need to bucket and dedupe by hash to get duplicates
BucketBy(BucketedBy.CRC, DedupeType.Full);
// Then follow the same tactics as before
@@ -3244,7 +3215,7 @@ namespace SabreTools.Library.DatFiles
/// <returns>True if split succeeded, false otherwise</returns>
private bool SplitByLevel(string outDir, bool shortname, bool basedat)
{
// First, organize by games so that we can do the right thing
// First, bucket by games so that we can do the right thing
BucketBy(BucketedBy.Game, DedupeType.None, lower: false, norename: true);
// Create a temporary DAT to add things to

14
SabreTools.Library/DatFiles/DatStats.cs
View File

@@ -258,33 +258,33 @@ namespace SabreTools.Library.DatFiles
/// </summary>
public BucketedBy GetBestAvailable()
{
// If all items are supposed to have a SHA-512, we sort by that
// If all items are supposed to have a SHA-512, we bucket by that
if (RomCount + DiskCount - NodumpCount == SHA512Count)
return BucketedBy.SHA512;
// If all items are supposed to have a SHA-384, we sort by that
// If all items are supposed to have a SHA-384, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA384Count)
return BucketedBy.SHA384;
// If all items are supposed to have a SHA-256, we sort by that
// If all items are supposed to have a SHA-256, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA256Count)
return BucketedBy.SHA256;
// If all items are supposed to have a SHA-1, we sort by that
// If all items are supposed to have a SHA-1, we bucket by that
else if (RomCount + DiskCount - NodumpCount == SHA1Count)
return BucketedBy.SHA1;
#if NET_FRAMEWORK
// If all items are supposed to have a RIPEMD160, we sort by that
// If all items are supposed to have a RIPEMD160, we bucket by that
else if (RomCount + DiskCount - NodumpCount == RIPEMD160Count)
return BucketedBy.RIPEMD160;
#endif
// If all items are supposed to have a MD5, we sort by that
// If all items are supposed to have a MD5, we bucket by that
else if (RomCount + DiskCount - NodumpCount == MD5Count)
return BucketedBy.MD5;
// Otherwise, we sort by CRC
// Otherwise, we bucket by CRC
else
return BucketedBy.CRC;
}

8
SabreTools.Library/DatFiles/Filter.cs
View File

@@ -499,7 +499,7 @@ namespace SabreTools.Library.DatFiles
{
Globals.Logger.User("Creating fully non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is sort by game
// For sake of ease, the first thing we want to do is bucket by game
datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
@@ -523,7 +523,7 @@ namespace SabreTools.Library.DatFiles
{
Globals.Logger.User("Creating merged sets from the DAT");
// For sake of ease, the first thing we want to do is sort by game
// For sake of ease, the first thing we want to do is bucket by game
datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
@@ -546,7 +546,7 @@ namespace SabreTools.Library.DatFiles
{
Globals.Logger.User("Creating non-merged sets from the DAT");
// For sake of ease, the first thing we want to do is sort by game
// For sake of ease, the first thing we want to do is bucket by game
datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information
@@ -569,7 +569,7 @@ namespace SabreTools.Library.DatFiles
{
Globals.Logger.User("Creating split sets from the DAT");
// For sake of ease, the first thing we want to do is sort by game
// For sake of ease, the first thing we want to do is bucket by game
datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true);
// Now we want to loop through all of the games and set the correct information

10
SabreTools.Library/DatItems/DatItem.cs
View File

@@ -971,19 +971,19 @@ namespace SabreTools.Library.DatItems
#region Sorting and Merging
/// <summary>
/// Get the dictionary key that should be used for a given item and sorting type
/// Get the dictionary key that should be used for a given item and bucketing type
/// </summary>
/// <param name="sortedBy">BucketedBy enum representing what key to get</param>
/// <param name="bucketedBy">BucketedBy enum representing what key to get</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
/// <returns>String representing the key to be used for the DatItem</returns>
public string GetKey(BucketedBy sortedBy, bool lower = true, bool norename = true)
public string GetKey(BucketedBy bucketedBy, bool lower = true, bool norename = true)
{
// Set the output key as the default blank string
string key = string.Empty;
// Now determine what the key should be based on the sortedBy value
switch (sortedBy)
// Now determine what the key should be based on the bucketedBy value
switch (bucketedBy)
{
case BucketedBy.CRC:
key = (this.ItemType == ItemType.Rom ? ((Rom)this).CRC : Constants.CRCZero);

4
SabreTools/SabreTools.Help.cs
View File

@@ -3019,9 +3019,9 @@ The following systems have headers that this program can work with:
// If we have the depot flag, respect it
if (depot)
datdata.RebuildDepot(Inputs, Path.Combine(outDir, datdata.GetFileName()), date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst);
datdata.RebuildDepot(Inputs, Path.Combine(outDir, datdata.DatHeader.FileName), date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst);
else
datdata.RebuildGeneric(Inputs, Path.Combine(outDir, datdata.GetFileName()), quickScan, date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst, chdsAsFiles);
datdata.RebuildGeneric(Inputs, Path.Combine(outDir, datdata.DatHeader.FileName), quickScan, date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst, chdsAsFiles);
}
}