diff --git a/SabreTools.Helper/Data/Enums.cs b/SabreTools.Helper/Data/Enums.cs index 4faa91b1..0decc2cc 100644 --- a/SabreTools.Helper/Data/Enums.cs +++ b/SabreTools.Helper/Data/Enums.cs @@ -33,6 +33,19 @@ Unzip, } + /// + /// Determines how the current dictionary is sorted by + /// + public enum SortedBy + { + Default = 0, + Size, + CRC, + MD5, + SHA1, + Game, + } + #endregion #region DatItem related diff --git a/SabreTools.Helper/Objects/Dat/DatFile.cs b/SabreTools.Helper/Objects/Dat/DatFile.cs index 99f435fd..eefa076e 100644 --- a/SabreTools.Helper/Objects/Dat/DatFile.cs +++ b/SabreTools.Helper/Objects/Dat/DatFile.cs @@ -37,6 +37,7 @@ namespace SabreTools.Helper private bool _excludeOf; private bool _mergeRoms; private SortedDictionary> _files; + private SortedBy _sortedBy; // Data specific to the Miss DAT type private bool _useGame; @@ -179,6 +180,11 @@ namespace SabreTools.Helper _files = value; } } + public SortedBy SortedBy + { + get { return _sortedBy; } + set { _sortedBy = value; } + } // Data specific to the Miss DAT type public bool UseGame @@ -271,123 +277,6 @@ namespace SabreTools.Helper #endregion - #region Constructors - - /// - /// Create a default, empty Dat object - /// - public DatFile() - { - // Nothing needs to be done - } - - /// - /// Create a new Dat object with the included information (standard Dats) - /// - /// New filename - /// New name - /// New description - /// New rootdir - /// New category - /// New version - /// New date - /// New author - /// New email - /// New homepage - /// New URL - /// New comment - /// New header - /// True to set SuperDAT type, false otherwise - /// None, Split, Full - /// None, Obsolete, Required, Ignore - /// None, Zip, Unzip - /// Non-zero flag for output format, zero otherwise for default - /// True to dedupe the roms in the DAT, false otherwise (default) - /// SortedDictionary of lists of DatItem objects - public DatFile(string fileName, string name, string description, string rootDir, string category, string version, string date, - string author, string email, string homepage, string url, string comment, string header, string type, ForceMerging forceMerging, - ForceNodump forceNodump, ForcePacking forcePacking, OutputFormat outputFormat, bool mergeRoms, SortedDictionary> files) - { - _fileName = fileName; - _name = name; - _description = description; - _rootDir = rootDir; - _category = category; - _version = version; - _date = date; - _author = author; - _email = email; - _homepage = homepage; - _url = url; - _comment = comment; - _header = header; - _type = type; - _forceMerging = forceMerging; - _forceNodump = forceNodump; - _forcePacking = forcePacking; - _outputFormat = outputFormat; - _mergeRoms = mergeRoms; - _files = files; - - _romCount = 0; - _diskCount = 0; - _totalSize = 0; - _crcCount = 0; - _md5Count = 0; - _sha1Count = 0; - _nodumpCount = 0; - } - - /// - /// Create a new Dat object with the included information (missfile) - /// - /// New filename - /// New name - /// New description - /// Non-zero flag for output format, zero otherwise for default - /// True to dedupe the roms in the DAT, false otherwise (default) - /// SortedDictionary of lists of DatItem objects - /// True if games are to be used in output, false if roms are - /// Generic prefix to be added to each line - /// Generic postfix to be added to each line - /// Add quotes to each item - /// Replace all extensions with another - /// Add an extension to all items - /// Remove all extensions - /// Add the dat name as a directory prefix - /// Output files in romba format - public DatFile(string fileName, string name, string description, OutputFormat outputFormat, bool mergeRoms, - SortedDictionary> files, bool useGame, string prefix, string postfix, bool quotes, - string repExt, string addExt, bool remExt, bool gameName, bool romba) - { - _fileName = fileName; - _name = name; - _description = description; - _outputFormat = outputFormat; - _mergeRoms = mergeRoms; - _files = files; - - _useGame = useGame; - _prefix = prefix; - _postfix = postfix; - _quotes = quotes; - _repExt = repExt; - _addExt = addExt; - _remExt = remExt; - _gameName = gameName; - _romba = romba; - - _romCount = 0; - _diskCount = 0; - _totalSize = 0; - _crcCount = 0; - _md5Count = 0; - _sha1Count = 0; - _nodumpCount = 0; - } - - #endregion - #region Instance Methods #region Bucketing @@ -401,7 +290,14 @@ namespace SabreTools.Helper /// True if the number of hashes counted is to be output (default), false otherwise public void BucketByGame(bool mergeroms, bool norename, Logger logger, bool output = true) { - logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + // If we already have the right sorting, trust it + if (_sortedBy == SortedBy.Game) + { + return; + } + + // Set the sorted type + _sortedBy = SortedBy.Game; SortedDictionary> sortable = new SortedDictionary>(); long count = 0; @@ -412,6 +308,8 @@ namespace SabreTools.Helper Files = sortable; } + logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + // Process each all of the roms List keys = Files.Keys.ToList(); foreach (string key in keys) @@ -468,6 +366,322 @@ namespace SabreTools.Helper Files = sortable; } + /// + /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by Size + /// + /// True if roms should be deduped, false otherwise + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + public void BucketBySize(bool mergeroms, Logger logger, bool output = true) + { + // If we already have the right sorting, trust it + if (_sortedBy == SortedBy.Size) + { + return; + } + + // Set the sorted type + _sortedBy = SortedBy.Size; + + SortedDictionary> sortable = new SortedDictionary>(); + long count = 0; + + // If we have a null dict or an empty one, output a new dictionary + if (Files == null || Files.Count == 0) + { + Files = sortable; + } + + logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + + // Process each all of the roms + List keys = Files.Keys.ToList(); + foreach (string key in keys) + { + List roms = Files[key]; + + // If we're merging the roms, do so + if (mergeroms) + { + roms = DatItem.Merge(roms, logger); + } + + // Now add each of the roms to their respective games + foreach (DatItem rom in roms) + { + count++; + string newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).Size.ToString() : "-1"); + if (sortable.ContainsKey(newkey)) + { + sortable[newkey].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + sortable.Add(newkey, temp); + } + } + } + + // Now go through and sort all of the lists + keys = sortable.Keys.ToList(); + foreach (string key in keys) + { + List sortedlist = sortable[key]; + DatItem.Sort(ref sortedlist, false); + sortable[key] = sortedlist; + } + + // Output the count if told to + if (output) + { + logger.User("A total of " + count + " file hashes will be written out to file"); + } + + // Now assign the dictionary back + Files = sortable; + } + + /// + /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by CRC + /// + /// True if roms should be deduped, false otherwise + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + public void BucketByCRC(bool mergeroms, Logger logger, bool output = true) + { + // If we already have the right sorting, trust it + if (_sortedBy == SortedBy.CRC) + { + return; + } + + // Set the sorted type + _sortedBy = SortedBy.CRC; + + SortedDictionary> sortable = new SortedDictionary>(); + long count = 0; + + // If we have a null dict or an empty one, output a new dictionary + if (Files == null || Files.Count == 0) + { + Files = sortable; + } + + logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + + // Process each all of the roms + List keys = Files.Keys.ToList(); + foreach (string key in keys) + { + List roms = Files[key]; + + // If we're merging the roms, do so + if (mergeroms) + { + roms = DatItem.Merge(roms, logger); + } + + // Now add each of the roms to their respective games + foreach (DatItem rom in roms) + { + count++; + string newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); + if (sortable.ContainsKey(newkey)) + { + sortable[newkey].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + sortable.Add(newkey, temp); + } + } + } + + // Now go through and sort all of the lists + keys = sortable.Keys.ToList(); + foreach (string key in keys) + { + List sortedlist = sortable[key]; + DatItem.Sort(ref sortedlist, false); + sortable[key] = sortedlist; + } + + // Output the count if told to + if (output) + { + logger.User("A total of " + count + " file hashes will be written out to file"); + } + + // Now assign the dictionary back + Files = sortable; + } + + /// + /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by MD5 + /// + /// True if roms should be deduped, false otherwise + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + public void BucketByMD5(bool mergeroms, Logger logger, bool output = true) + { + // If we already have the right sorting, trust it + if (_sortedBy == SortedBy.MD5) + { + return; + } + + // Set the sorted type + _sortedBy = SortedBy.MD5; + + SortedDictionary> sortable = new SortedDictionary>(); + long count = 0; + + // If we have a null dict or an empty one, output a new dictionary + if (Files == null || Files.Count == 0) + { + Files = sortable; + } + + logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + + // Process each all of the roms + List keys = Files.Keys.ToList(); + foreach (string key in keys) + { + List roms = Files[key]; + + // If we're merging the roms, do so + if (mergeroms) + { + roms = DatItem.Merge(roms, logger); + } + + // Now add each of the roms to their respective games + foreach (DatItem rom in roms) + { + count++; + string newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).MD5 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).MD5 + : Constants.MD5Zero)); + if (sortable.ContainsKey(newkey)) + { + sortable[newkey].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + sortable.Add(newkey, temp); + } + } + } + + // Now go through and sort all of the lists + keys = sortable.Keys.ToList(); + foreach (string key in keys) + { + List sortedlist = sortable[key]; + DatItem.Sort(ref sortedlist, false); + sortable[key] = sortedlist; + } + + // Output the count if told to + if (output) + { + logger.User("A total of " + count + " file hashes will be written out to file"); + } + + // Now assign the dictionary back + Files = sortable; + } + + /// + /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by SHA1 + /// + /// True if roms should be deduped, false otherwise + /// Logger object for file and console output + /// True if the number of hashes counted is to be output (default), false otherwise + public void BucketBySHA1(bool mergeroms, Logger logger, bool output = true) + { + // If we already have the right sorting, trust it + if (_sortedBy == SortedBy.SHA1) + { + return; + } + + // Set the sorted type + _sortedBy = SortedBy.SHA1; + + SortedDictionary> sortable = new SortedDictionary>(); + long count = 0; + + // If we have a null dict or an empty one, output a new dictionary + if (Files == null || Files.Count == 0) + { + Files = sortable; + } + + logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output"); + + // Process each all of the roms + List keys = Files.Keys.ToList(); + foreach (string key in keys) + { + List roms = Files[key]; + + // If we're merging the roms, do so + if (mergeroms) + { + roms = DatItem.Merge(roms, logger); + } + + // Now add each of the roms to their respective games + foreach (DatItem rom in roms) + { + count++; + string newkey = (rom.Type == ItemType.Rom + ? ((Rom)rom).SHA1 + : (rom.Type == ItemType.Disk + ? ((Disk)rom).SHA1 + : Constants.MD5Zero)); + if (sortable.ContainsKey(newkey)) + { + sortable[newkey].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + sortable.Add(newkey, temp); + } + } + } + + // Now go through and sort all of the lists + keys = sortable.Keys.ToList(); + foreach (string key in keys) + { + List sortedlist = sortable[key]; + DatItem.Sort(ref sortedlist, false); + sortable[key] = sortedlist; + } + + // Output the count if told to + if (output) + { + logger.User("A total of " + count + " file hashes will be written out to file"); + } + + // Now assign the dictionary back + Files = sortable; + } + #endregion #region Cloning Methods @@ -476,44 +690,45 @@ namespace SabreTools.Helper { return new DatFile { - FileName = this.FileName, - Name = this.Name, - Description = this.Description, - RootDir = this.RootDir, - Category = this.Category, - Version = this.Version, - Date = this.Date, - Author = this.Author, - Email = this.Email, - Homepage = this.Homepage, - Url = this.Url, - Comment = this.Comment, - Header = this.Header, - Type = this.Type, - ForceMerging = this.ForceMerging, - ForceNodump = this.ForceNodump, - ForcePacking = this.ForcePacking, - ExcludeOf = this.ExcludeOf, - OutputFormat = this.OutputFormat, - MergeRoms = this.MergeRoms, - Files = this.Files, - UseGame = this.UseGame, - Prefix = this.Prefix, - Postfix = this.Postfix, - Quotes = this.Quotes, - RepExt = this.RepExt, - AddExt = this.AddExt, - RemExt = this.RemExt, - GameName = this.GameName, - Romba = this.Romba, - RomCount = this.RomCount, - DiskCount = this.DiskCount, - TotalSize = this.TotalSize, - CRCCount = this.CRCCount, - MD5Count = this.MD5Count, - SHA1Count = this.SHA1Count, - BaddumpCount = this.BaddumpCount, - NodumpCount = this.NodumpCount, + FileName = _fileName, + Name = _name, + Description = _description, + RootDir = _rootDir, + Category = _category, + Version = _version, + Date = _date, + Author = _author, + Email = _email, + Homepage = _homepage, + Url = _url, + Comment = _comment, + Header = _header, + Type = _type, + ForceMerging = _forceMerging, + ForceNodump = _forceNodump, + ForcePacking = _forcePacking, + ExcludeOf = _excludeOf, + OutputFormat = _outputFormat, + MergeRoms = _mergeRoms, + Files = _files, + SortedBy = _sortedBy, + UseGame = _useGame, + Prefix = _prefix, + Postfix = _postfix, + Quotes = _quotes, + RepExt = _repExt, + AddExt = _addExt, + RemExt = _remExt, + GameName = _gameName, + Romba = _romba, + RomCount = _romCount, + DiskCount = _diskCount, + TotalSize = _totalSize, + CRCCount = _crcCount, + MD5Count = _md5Count, + SHA1Count = _sha1Count, + BaddumpCount = _baddumpCount, + NodumpCount = _nodumpCount, }; } @@ -521,36 +736,37 @@ namespace SabreTools.Helper { return new DatFile { - FileName = this.FileName, - Name = this.Name, - Description = this.Description, - RootDir = this.RootDir, - Category = this.Category, - Version = this.Version, - Date = this.Date, - Author = this.Author, - Email = this.Email, - Homepage = this.Homepage, - Url = this.Url, - Comment = this.Comment, - Header = this.Header, - Type = this.Type, - ForceMerging = this.ForceMerging, - ForceNodump = this.ForceNodump, - ForcePacking = this.ForcePacking, - ExcludeOf = this.ExcludeOf, - OutputFormat = this.OutputFormat, - MergeRoms = this.MergeRoms, + FileName = _fileName, + Name = _name, + Description = _description, + RootDir = _rootDir, + Category = _category, + Version = _version, + Date = _date, + Author = _author, + Email = _email, + Homepage = _homepage, + Url = _url, + Comment = _comment, + Header = _header, + Type = _type, + ForceMerging = _forceMerging, + ForceNodump = _forceNodump, + ForcePacking = _forcePacking, + ExcludeOf = _excludeOf, + OutputFormat = _outputFormat, + MergeRoms = _mergeRoms, Files = new SortedDictionary>(), - UseGame = this.UseGame, - Prefix = this.Prefix, - Postfix = this.Postfix, - Quotes = this.Quotes, - RepExt = this.RepExt, - AddExt = this.AddExt, - RemExt = this.RemExt, - GameName = this.GameName, - Romba = this.Romba, + SortedBy = SortedBy.Default, + UseGame = _useGame, + Prefix = _prefix, + Postfix = _postfix, + Quotes = _quotes, + RepExt = _repExt, + AddExt = _addExt, + RemExt = _remExt, + GameName = _gameName, + Romba = _romba, }; } @@ -3195,7 +3411,14 @@ namespace SabreTools.Helper */ string[] rominfo = line.Split('¬'); - Rom rom = new Rom(rominfo[5], Int64.Parse(rominfo[7]), rominfo[6], null, null, ItemStatus.None, null, rominfo[3], null, + // Try getting the size separately + long size = 0; + if (!Int64.TryParse(rominfo[7], out size)) + { + size = 0; + } + + Rom rom = new Rom(rominfo[5], size, rominfo[6], null, null, ItemStatus.None, null, rominfo[3], null, rominfo[4], null, null, rominfo[8], rominfo[1], null, null, false, null, null, sysid, null, srcid, null); // Now process and add the rom diff --git a/SabreTools.Helper/Objects/Dat/DatItem.cs b/SabreTools.Helper/Objects/Dat/DatItem.cs index be5d33fa..7c9bdab9 100644 --- a/SabreTools.Helper/Objects/Dat/DatItem.cs +++ b/SabreTools.Helper/Objects/Dat/DatItem.cs @@ -623,30 +623,70 @@ namespace SabreTools.Helper return output; } - // Try to find duplicates - List keys = datdata.Files.Keys.ToList(); - foreach (string key in keys) + // Get the correct dictionary based on what is available + string key = ""; + if (_itemType == ItemType.Rom && ((Rom)this).CRC != null) { - List roms = datdata.Files[key]; - List left = new List(); + key = ((Rom)this).CRC; + datdata.BucketByCRC(false, logger, false); + } + else if (_itemType == ItemType.Rom && ((Rom)this).MD5 != null) + { + key = ((Rom)this).MD5; + datdata.BucketByMD5(false, logger, false); + } + else if (_itemType == ItemType.Disk && ((Disk)this).MD5 != null) + { + key = ((Disk)this).MD5; + datdata.BucketByMD5(false, logger, false); + } + else if (_itemType == ItemType.Rom && ((Rom)this).SHA1 != null) + { + key = ((Rom)this).SHA1; + datdata.BucketBySHA1(false, logger, false); + } + else if (_itemType == ItemType.Disk && ((Disk)this).SHA1 != null) + { + key = ((Disk)this).SHA1; + datdata.BucketBySHA1(false, logger, false); + } + else if (_itemType == ItemType.Rom) + { + key = ((Rom)this).Size.ToString(); + datdata.BucketBySize(false, logger, false); + } + else + { + key = "-1"; + datdata.BucketBySize(false, logger, false); + } - foreach (DatItem rom in roms) - { - if (IsDuplicate(rom, logger)) - { - output.Add(rom); - } - else - { - left.Add(rom); - } - } + // If the key doesn't exist, return the empty list + if (!datdata.Files.ContainsKey(key)) + { + return output; + } - // If we're in removal mode, replace the list with the new one - if (remove) + // Try to find duplicates + List roms = datdata.Files[key]; + List left = new List(); + + foreach (DatItem rom in roms) + { + if (IsDuplicate(rom, logger)) { - datdata.Files[key] = left; + output.Add(rom); } + else + { + left.Add(rom); + } + } + + // If we're in removal mode, replace the list with the new one + if (remove) + { + datdata.Files[key] = left; } return output;