diff --git a/RombaSharp/RombaSharp.Help.cs b/RombaSharp/RombaSharp.Help.cs index 2002bc06..3e572dd3 100644 --- a/RombaSharp/RombaSharp.Help.cs +++ b/RombaSharp/RombaSharp.Help.cs @@ -1595,6 +1595,7 @@ contents of any changed dats."; unneeded.Add(hash); } } + datroot.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); watch.Stop(); @@ -1691,6 +1692,7 @@ contents of any changed dats."; // Now rescan the depot itself DatFile depot = DatFile.Create(); + // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually depot.PopulateFromDir(depotname, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null); depot.BucketBy(BucketedBy.SHA1, DedupeType.None); diff --git a/RombaSharp/RombaSharp.Helpers.cs b/RombaSharp/RombaSharp.Helpers.cs index eb14d870..bf6e367f 100644 --- a/RombaSharp/RombaSharp.Helpers.cs +++ b/RombaSharp/RombaSharp.Helpers.cs @@ -252,97 +252,102 @@ namespace RombaSharp DatFile tempdat = DatFile.CreateAndParse(fullpath); // If the Dat wasn't empty, add the information - SqliteCommand slc; - if (tempdat.GetCount() != 0) + SqliteCommand slc = null; + string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; + string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; + string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES"; + string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; + string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; + + // Loop through the parsed entries + bool hasItems = false; + foreach (string romkey in tempdat.Keys) { - string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; - string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; - string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES"; - string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; - string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; - - // Loop through the parsed entries - foreach (string romkey in tempdat.Keys) + foreach (DatItem datItem in tempdat[romkey]) { - foreach (DatItem datItem in tempdat[romkey]) - { - Globals.Logger.Verbose($"Checking and adding file '{datItem.Name}'"); + Globals.Logger.Verbose($"Checking and adding file '{datItem.Name}'"); - if (datItem.ItemType == ItemType.Rom) + if (datItem.ItemType == ItemType.Rom) + { + Rom rom = (Rom)datItem; + hasItems = true; + + if (!string.IsNullOrWhiteSpace(rom.CRC)) + crcquery += $" (\"{rom.CRC}\"),"; + + if (!string.IsNullOrWhiteSpace(rom.MD5)) + md5query += $" (\"{rom.MD5}\"),"; + + if (!string.IsNullOrWhiteSpace(rom.SHA1)) { - Rom rom = (Rom)datItem; + sha1query += $" (\"{rom.SHA1}\"),"; if (!string.IsNullOrWhiteSpace(rom.CRC)) - crcquery += $" (\"{rom.CRC}\"),"; - + crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),"; + if (!string.IsNullOrWhiteSpace(rom.MD5)) - md5query += $" (\"{rom.MD5}\"),"; - - if (!string.IsNullOrWhiteSpace(rom.SHA1)) - { - sha1query += $" (\"{rom.SHA1}\"),"; - - if (!string.IsNullOrWhiteSpace(rom.CRC)) - crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),"; - - if (!string.IsNullOrWhiteSpace(rom.MD5)) - md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),"; - } + md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),"; } - else if (datItem.ItemType == ItemType.Disk) + } + else if (datItem.ItemType == ItemType.Disk) + { + Disk disk = (Disk)datItem; + hasItems = true; + + if (!string.IsNullOrWhiteSpace(disk.MD5)) + md5query += $" (\"{disk.MD5}\"),"; + + if (!string.IsNullOrWhiteSpace(disk.SHA1)) { - Disk disk = (Disk)datItem; + sha1query += $" (\"{disk.SHA1}\"),"; if (!string.IsNullOrWhiteSpace(disk.MD5)) - md5query += $" (\"{disk.MD5}\"),"; - - if (!string.IsNullOrWhiteSpace(disk.SHA1)) - { - sha1query += $" (\"{disk.SHA1}\"),"; - - if (!string.IsNullOrWhiteSpace(disk.MD5)) - md5sha1query += $" (\"{disk.MD5}\", \"{disk.SHA1}\"),"; - } + md5sha1query += $" (\"{disk.MD5}\", \"{disk.SHA1}\"),"; } } } - - // Now run the queries after fixing them - if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") - { - slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); - slc.ExecuteNonQuery(); - } - - if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") - { - slc = new SqliteCommand(md5query.TrimEnd(','), dbc); - slc.ExecuteNonQuery(); - } - - if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES") - { - slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); - slc.ExecuteNonQuery(); - } - - if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") - { - slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); - slc.ExecuteNonQuery(); - } - - if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") - { - slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); - slc.ExecuteNonQuery(); - } } - string datquery = $"INSERT OR IGNORE INTO dat (hash) VALUES (\"{dat.SHA1}\")"; - slc = new SqliteCommand(datquery, dbc); - slc.ExecuteNonQuery(); - slc.Dispose(); + // Now run the queries after fixing them + if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") + { + slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + + if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") + { + slc = new SqliteCommand(md5query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + + if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES") + { + slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + + if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") + { + slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + + if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") + { + slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + + // Only add the DAT if it's non-empty + if (hasItems) + { + string datquery = $"INSERT OR IGNORE INTO dat (hash) VALUES (\"{dat.SHA1}\")"; + slc = new SqliteCommand(datquery, dbc); + slc.ExecuteNonQuery(); + } + + slc?.Dispose(); } #endregion diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index bbda9fef..bbd33c5d 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -35,7 +35,7 @@ namespace SabreTools.Library.DatFiles internal DatStats DatStats = new DatStats(); /// - /// Determine the sorting key for all items + /// Determine the bucketing key for all items /// private BucketedBy BucketedBy; @@ -136,29 +136,13 @@ namespace SabreTools.Library.DatFiles return false; } - /// - /// Get total item count statistic - /// - public long GetCount() - { - return DatStats.Count; - } - - /// - /// Get the FileName header value - /// - public string GetFileName() - { - return DatHeader.FileName; - } - /// /// Get the keys from the file dictionary /// /// List of the keys public List Keys { - get { return Items.Keys.Select(item => (string)item.Clone()).ToList(); } + get { return Items.Keys.ToList(); } } /// @@ -198,19 +182,6 @@ namespace SabreTools.Library.DatFiles Items[key].Remove(value); } - /// - /// Remove a range of values from the file dictionary if they exists - /// - /// Key in the dictionary to remove from - /// Value to remove from the dictionary - public void RemoveRange(string key, List value) - { - foreach (DatItem item in value) - { - Remove(key, item); - } - } - /// /// Set the Date header value /// @@ -260,9 +231,9 @@ namespace SabreTools.Library.DatFiles #region Bucketing /// - /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method + /// Take the arbitrarily bucketed Files Dictionary and convert to one bucketed by a user-defined method /// - /// BucketedBy enum representing how to sort the individual items + /// BucketedBy enum representing how to bucket the individual items /// Dedupe type that should be used /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted @@ -831,7 +802,7 @@ namespace SabreTools.Library.DatFiles // If we are matching based on DatItem fields of any sort if (updateFields.Intersect(datItemFields).Any()) { - // For comparison's sake, we want to use CRC as the base ordering + // For comparison's sake, we want to use CRC as the base bucketing BucketBy(BucketedBy.CRC, DedupeType.Full); intDat.BucketBy(BucketedBy.CRC, DedupeType.None); @@ -1138,7 +1109,7 @@ namespace SabreTools.Library.DatFiles // If we are matching based on Machine fields of any sort if (updateFields.Intersect(machineFields).Any()) { - // For comparison's sake, we want to use Machine Name as the base ordering + // For comparison's sake, we want to use Machine Name as the base bucketing BucketBy(BucketedBy.Game, DedupeType.Full); intDat.BucketBy(BucketedBy.Game, DedupeType.None); @@ -1253,7 +1224,7 @@ namespace SabreTools.Library.DatFiles DatFile intDat = Create(); intDat.Parse(path, 1, keep: true); - // For comparison's sake, we want to use CRC as the base ordering + // For comparison's sake, we want to use CRC as the base bucketing intDat.BucketBy(BucketedBy.CRC, DedupeType.Full); // Then we do a hashwise comparison against the base DAT @@ -1326,7 +1297,7 @@ namespace SabreTools.Library.DatFiles outDats = outDatsArray.ToList(); watch.Stop(); - // Then, ensure that the internal dat can be sorted in the best possible way + // Then, ensure that the internal dat can be bucketed in the best possible way BucketBy(BucketedBy.CRC, DedupeType.None); // Now, loop through the dictionary and populate the correct DATs @@ -2253,7 +2224,7 @@ namespace SabreTools.Library.DatFiles if (directories.Count == 0) return success; - // Now that we have a list of depots, we want to sort the input DAT by SHA-1 + // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 BucketBy(BucketedBy.SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild @@ -2827,7 +2798,7 @@ namespace SabreTools.Library.DatFiles if (directories.Count == 0) return success; - // Now that we have a list of depots, we want to sort the input DAT by SHA-1 + // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 BucketBy(BucketedBy.SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild @@ -2918,7 +2889,7 @@ namespace SabreTools.Library.DatFiles // If we are checking hashes only, essentially diff the inputs if (hashOnly) { - // First we need to sort and dedupe by hash to get duplicates + // First we need to bucket and dedupe by hash to get duplicates BucketBy(BucketedBy.CRC, DedupeType.Full); // Then follow the same tactics as before @@ -3244,7 +3215,7 @@ namespace SabreTools.Library.DatFiles /// True if split succeeded, false otherwise private bool SplitByLevel(string outDir, bool shortname, bool basedat) { - // First, organize by games so that we can do the right thing + // First, bucket by games so that we can do the right thing BucketBy(BucketedBy.Game, DedupeType.None, lower: false, norename: true); // Create a temporary DAT to add things to diff --git a/SabreTools.Library/DatFiles/DatStats.cs b/SabreTools.Library/DatFiles/DatStats.cs index 9a3179e4..8b5abf26 100644 --- a/SabreTools.Library/DatFiles/DatStats.cs +++ b/SabreTools.Library/DatFiles/DatStats.cs @@ -258,33 +258,33 @@ namespace SabreTools.Library.DatFiles /// public BucketedBy GetBestAvailable() { - // If all items are supposed to have a SHA-512, we sort by that + // If all items are supposed to have a SHA-512, we bucket by that if (RomCount + DiskCount - NodumpCount == SHA512Count) return BucketedBy.SHA512; - // If all items are supposed to have a SHA-384, we sort by that + // If all items are supposed to have a SHA-384, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA384Count) return BucketedBy.SHA384; - // If all items are supposed to have a SHA-256, we sort by that + // If all items are supposed to have a SHA-256, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA256Count) return BucketedBy.SHA256; - // If all items are supposed to have a SHA-1, we sort by that + // If all items are supposed to have a SHA-1, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA1Count) return BucketedBy.SHA1; #if NET_FRAMEWORK - // If all items are supposed to have a RIPEMD160, we sort by that + // If all items are supposed to have a RIPEMD160, we bucket by that else if (RomCount + DiskCount - NodumpCount == RIPEMD160Count) return BucketedBy.RIPEMD160; #endif - // If all items are supposed to have a MD5, we sort by that + // If all items are supposed to have a MD5, we bucket by that else if (RomCount + DiskCount - NodumpCount == MD5Count) return BucketedBy.MD5; - // Otherwise, we sort by CRC + // Otherwise, we bucket by CRC else return BucketedBy.CRC; } diff --git a/SabreTools.Library/DatFiles/Filter.cs b/SabreTools.Library/DatFiles/Filter.cs index cc5f13cd..c2cc7555 100644 --- a/SabreTools.Library/DatFiles/Filter.cs +++ b/SabreTools.Library/DatFiles/Filter.cs @@ -499,7 +499,7 @@ namespace SabreTools.Library.DatFiles { Globals.Logger.User("Creating fully non-merged sets from the DAT"); - // For sake of ease, the first thing we want to do is sort by game + // For sake of ease, the first thing we want to do is bucket by game datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information @@ -523,7 +523,7 @@ namespace SabreTools.Library.DatFiles { Globals.Logger.User("Creating merged sets from the DAT"); - // For sake of ease, the first thing we want to do is sort by game + // For sake of ease, the first thing we want to do is bucket by game datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information @@ -546,7 +546,7 @@ namespace SabreTools.Library.DatFiles { Globals.Logger.User("Creating non-merged sets from the DAT"); - // For sake of ease, the first thing we want to do is sort by game + // For sake of ease, the first thing we want to do is bucket by game datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information @@ -569,7 +569,7 @@ namespace SabreTools.Library.DatFiles { Globals.Logger.User("Creating split sets from the DAT"); - // For sake of ease, the first thing we want to do is sort by game + // For sake of ease, the first thing we want to do is bucket by game datFile.BucketBy(BucketedBy.Game, mergeroms, norename: true); // Now we want to loop through all of the games and set the correct information diff --git a/SabreTools.Library/DatItems/DatItem.cs b/SabreTools.Library/DatItems/DatItem.cs index ae16af3f..2bb7d6d2 100644 --- a/SabreTools.Library/DatItems/DatItem.cs +++ b/SabreTools.Library/DatItems/DatItem.cs @@ -971,19 +971,19 @@ namespace SabreTools.Library.DatItems #region Sorting and Merging /// - /// Get the dictionary key that should be used for a given item and sorting type + /// Get the dictionary key that should be used for a given item and bucketing type /// - /// BucketedBy enum representing what key to get + /// BucketedBy enum representing what key to get /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted /// String representing the key to be used for the DatItem - public string GetKey(BucketedBy sortedBy, bool lower = true, bool norename = true) + public string GetKey(BucketedBy bucketedBy, bool lower = true, bool norename = true) { // Set the output key as the default blank string string key = string.Empty; - // Now determine what the key should be based on the sortedBy value - switch (sortedBy) + // Now determine what the key should be based on the bucketedBy value + switch (bucketedBy) { case BucketedBy.CRC: key = (this.ItemType == ItemType.Rom ? ((Rom)this).CRC : Constants.CRCZero); diff --git a/SabreTools/SabreTools.Help.cs b/SabreTools/SabreTools.Help.cs index a98c4397..cd2e15db 100644 --- a/SabreTools/SabreTools.Help.cs +++ b/SabreTools/SabreTools.Help.cs @@ -3019,9 +3019,9 @@ The following systems have headers that this program can work with: // If we have the depot flag, respect it if (depot) - datdata.RebuildDepot(Inputs, Path.Combine(outDir, datdata.GetFileName()), date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst); + datdata.RebuildDepot(Inputs, Path.Combine(outDir, datdata.DatHeader.FileName), date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst); else - datdata.RebuildGeneric(Inputs, Path.Combine(outDir, datdata.GetFileName()), quickScan, date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst, chdsAsFiles); + datdata.RebuildGeneric(Inputs, Path.Combine(outDir, datdata.DatHeader.FileName), quickScan, date, delete, inverse, outputFormat, updateDat, headerToCheckAgainst, chdsAsFiles); } }