using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Web; using SabreTools.Library.Data; #if MONO using System.IO; #else using Alphaleonis.Win32.Filesystem; #endif namespace SabreTools.Library.Dats { public partial class DatFile { #region Instance Methods #region Bucketing /// /// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method /// /// SortedBy enum representing how to sort the individual items /// True if roms should be deduped, false otherwise /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true) { // If we already have the right sorting, trust it if (_sortedBy == bucketBy) { return; } // If we have a situation where there's no dictionary or no keys at all, we skip if (_files == null || _files.Count == 0) { return; } // Set the sorted type _sortedBy = bucketBy; // Create the temporary dictionary to sort into SortedDictionary> sortable = new SortedDictionary>(); Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : "")); // First do the initial sort of all of the roms List keys = Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { List roms = this[key]; // Now add each of the roms to their respective games foreach (DatItem rom in roms) { string newkey = ""; // We want to get the key most appropriate for the given sorting type switch (bucketBy) { case SortedBy.CRC: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero); break; case SortedBy.Game: newkey = (norename ? "" : rom.SystemID.ToString().PadLeft(10, '0') + "-" + rom.SourceID.ToString().PadLeft(10, '0') + "-") + (String.IsNullOrEmpty(rom.Machine.Name) ? "Default" : rom.Machine.Name); if (lower) { newkey = newkey.ToLowerInvariant(); } if (newkey == null) { newkey = "null"; } newkey = HttpUtility.HtmlEncode(newkey); break; case SortedBy.MD5: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).MD5 : (rom.Type == ItemType.Disk ? ((Disk)rom).MD5 : Constants.MD5Zero)); break; case SortedBy.SHA1: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).SHA1 : (rom.Type == ItemType.Disk ? ((Disk)rom).SHA1 : Constants.SHA1Zero)); break; case SortedBy.SHA256: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).SHA256 : (rom.Type == ItemType.Disk ? ((Disk)rom).SHA256 : Constants.SHA256Zero)); break; case SortedBy.SHA384: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).SHA384 : (rom.Type == ItemType.Disk ? ((Disk)rom).SHA384 : Constants.SHA384Zero)); break; case SortedBy.SHA512: newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).SHA512 : (rom.Type == ItemType.Disk ? ((Disk)rom).SHA512 : Constants.SHA512Zero)); break; } // Double and triple check the key if (newkey == null) { newkey = ""; } // Add the DatItem to the temp dictionary lock (sortable) { if (!sortable.ContainsKey(newkey)) { sortable.Add(newkey, new List()); } sortable[newkey].Add(rom); } } }); // Now go through and sort all of the individual lists keys = sortable.Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Get the possibly unsorted list List sortedlist = sortable[key]; // Sort the list of items to be consistent DatItem.Sort(ref sortedlist, false); // If we're merging the roms, do so if (mergeroms) { sortedlist = DatItem.Merge(sortedlist); } // Add the list back to the temp dictionary lock (sortable) { sortable[key] = sortedlist; } }); // Now assign the dictionary back _files = sortable; } #endregion #region Filtering /// /// Filter a DAT based on input parameters and modify the items /// /// Filter object for passing to the DatItem level /// True if we are supposed to trim names to NTFS length, false otherwise /// True if all games should be replaced by '!', false otherwise /// String representing root directory to compare against for length calculation public void Filter(Filter filter, bool single, bool trim, string root) { // Loop over every key in the dictionary List keys = Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { // For every item in the current key List items = this[key]; List newitems = new List(); foreach (DatItem item in items) { // If the rom passes the filter, include it if (filter.ItemPasses(item)) { // If we are in single game mode, rename all games if (single) { item.Machine.UpdateName("!"); } // If we are in NTFS trim mode, trim the game name if (trim) { // Windows max name length is 260 int usableLength = 260 - item.Machine.Name.Length - root.Length; if (item.Name.Length > usableLength) { string ext = Path.GetExtension(item.Name); item.Name = item.Name.Substring(0, usableLength - ext.Length); item.Name += ext; } } // Lock the list and add the item back lock (newitems) { newitems.Add(item); } } } Remove(key); AddRange(key, newitems); }); } /// /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof /// public void MachineDescriptionToName() { try { // First we want to get a mapping for all games to description ConcurrentDictionary mapping = new ConcurrentDictionary(); List keys = Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; foreach (DatItem item in items) { // If the key mapping doesn't exist, add it if (!mapping.ContainsKey(item.Machine.Name)) { mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''")); } } }); // Now we loop through every item and update accordingly keys = Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; List newItems = new List(); foreach (DatItem item in items) { // Update machine name if (!String.IsNullOrEmpty(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name)) { item.Machine.UpdateName(mapping[item.Machine.Name]); } // Update cloneof if (!String.IsNullOrEmpty(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf)) { item.Machine.UpdateCloneOf(mapping[item.Machine.CloneOf]); } // Update romof if (!String.IsNullOrEmpty(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf)) { item.Machine.UpdateRomOf(mapping[item.Machine.RomOf]); } // Update sampleof if (!String.IsNullOrEmpty(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf)) { item.Machine.UpdateSampleOf(mapping[item.Machine.SampleOf]); } // Add the new item to the output list items.Add(item); } // Replace the old list of roms with the new one Remove(key); AddRange(key, newItems); }); } catch (Exception ex) { Globals.Logger.Warning(ex.ToString()); } } /// /// Strip the given hash types from the DAT /// public void StripHashesFromItems() { // Output the logging statement Globals.Logger.User("Stripping requested hashes"); // Now process all of the roms List keys = Keys.ToList(); Parallel.ForEach(keys, Globals.ParallelOptions, key => { List items = this[key]; for (int j = 0; j < items.Count; j++) { DatItem item = items[j]; if (item.Type == ItemType.Rom) { Rom rom = (Rom)item; if ((StripHash & Hash.MD5) != 0) { rom.MD5 = null; } if ((StripHash & Hash.SHA1) != 0) { rom.SHA1 = null; } if ((StripHash & Hash.SHA256) != 0) { rom.SHA256 = null; } if ((StripHash & Hash.SHA384) != 0) { rom.SHA384 = null; } if ((StripHash & Hash.SHA512) != 0) { rom.SHA512 = null; } items[j] = rom; } else if (item.Type == ItemType.Disk) { Disk disk = (Disk)item; if ((StripHash & Hash.MD5) != 0) { disk.MD5 = null; } if ((StripHash & Hash.SHA1) != 0) { disk.SHA1 = null; } if ((StripHash & Hash.SHA256) != 0) { disk.SHA256 = null; } if ((StripHash & Hash.SHA384) != 0) { disk.SHA384 = null; } if ((StripHash & Hash.SHA512) != 0) { disk.SHA512 = null; } items[j] = disk; } } Remove(key); AddRange(key, items); }); } #endregion #region Merging/Splitting Methods /// /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets /// /// True if roms should be deduped, false otherwise public void CreateFullyNonMergedSets(bool mergeroms) { Globals.Logger.User("Creating fully non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game BucketBy(SortedBy.Game, mergeroms, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information AddRomsFromDevices(); AddRomsFromParent(); // Now that we have looped through the cloneof tags, we loop through the romof tags AddRomsFromBios(); // Then, remove the romof and cloneof tags so it's not picked up by the manager RemoveTagsFromChild(); // Finally, remove all sets that are labeled as bios or device //RemoveBiosAndDeviceSets(logger); } /// /// Use cloneof tags to create merged sets and remove the tags /// /// True if roms should be deduped, false otherwise public void CreateMergedSets(bool mergeroms) { Globals.Logger.User("Creating merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game BucketBy(SortedBy.Game, mergeroms, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information AddRomsToParent(); // Now that we have looped through the cloneof tags, we loop through the romof tags RemoveBiosRomsFromChild(); // Finally, remove the romof and cloneof tags so it's not picked up by the manager RemoveTagsFromChild(); } /// /// Use cloneof tags to create non-merged sets and remove the tags /// /// True if roms should be deduped, false otherwise public void CreateNonMergedSets(bool mergeroms) { Globals.Logger.User("Creating non-merged sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game BucketBy(SortedBy.Game, mergeroms, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information AddRomsFromParent(); // Now that we have looped through the cloneof tags, we loop through the romof tags RemoveBiosRomsFromChild(); // Finally, remove the romof and cloneof tags so it's not picked up by the manager RemoveTagsFromChild(); } /// /// Use cloneof and romof tags to create split sets and remove the tags /// /// True if roms should be deduped, false otherwise public void CreateSplitSets(bool mergeroms) { Globals.Logger.User("Creating split sets from the DAT"); // For sake of ease, the first thing we want to do is sort by game BucketBy(SortedBy.Game, mergeroms, norename: true); _sortedBy = SortedBy.Default; // Now we want to loop through all of the games and set the correct information RemoveRomsFromChild(); // Now that we have looped through the cloneof tags, we loop through the romof tags RemoveBiosRomsFromChild(); // Finally, remove the romof and cloneof tags so it's not picked up by the manager RemoveTagsFromChild(); } #endregion #region Merging/Splitting Helper Methods /// /// Use romof tags to add roms to the children /// private void AddRomsFromBios() { List games = Keys.ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue if (this[game].Count == 0) { continue; } // Determine if the game has a parent or not string parent = null; if (!String.IsNullOrEmpty(this[game][0].Machine.RomOf)) { parent = this[game][0].Machine.RomOf; } // If the parent doesnt exist, we want to continue if (String.IsNullOrEmpty(parent)) { continue; } // If the parent doesn't have any items, we want to continue if (this[parent].Count == 0) { continue; } // If the parent exists and has items, we copy the items from the parent to the current game Machine currentMachine = this[game][0].Machine; List parentItems = this[parent]; foreach (DatItem item in parentItems) { // Figure out the type of the item and add it accordingly switch (item.Type) { case ItemType.Archive: Archive archive = ((Archive)item).Clone() as Archive; archive.Machine = currentMachine; if (this[game].Where(i => i.Name == archive.Name).Count() == 0 && !this[game].Contains(archive)) { this[game].Add(archive); } break; case ItemType.BiosSet: BiosSet biosSet = ((BiosSet)item).Clone() as BiosSet; biosSet.Machine = currentMachine; if (this[game].Where(i => i.Name == biosSet.Name).Count() == 0 && !this[game].Contains(biosSet)) { this[game].Add(biosSet); } break; case ItemType.Disk: Disk disk = ((Disk)item).Clone() as Disk; disk.Machine = currentMachine; if (this[game].Where(i => i.Name == disk.Name).Count() == 0 && !this[game].Contains(disk)) { this[game].Add(disk); } break; case ItemType.Release: Release release = ((Release)item).Clone() as Release; release.Machine = currentMachine; if (this[game].Where(i => i.Name == release.Name).Count() == 0 && !this[game].Contains(release)) { this[game].Add(release); } break; case ItemType.Rom: Rom rom = ((Rom)item).Clone() as Rom; rom.Machine = currentMachine; if (this[game].Where(i => i.Name == rom.Name).Count() == 0 && !this[game].Contains(rom)) { this[game].Add(rom); } break; case ItemType.Sample: Sample sample = ((Sample)item).Clone() as Sample; sample.Machine = currentMachine; if (this[game].Where(i => i.Name == sample.Name).Count() == 0 && !this[game].Contains(sample)) { this[game].Add(sample); } break; } } } } /// /// Use device_ref tags to add roms to the children /// private void AddRomsFromDevices() { List games = Keys.ToList(); foreach (string game in games) { // If the game has no devices, we continue if (this[game][0].Machine.Devices == null || this[game][0].Machine.Devices.Count == 0) { continue; } // Determine if the game has any devices or not List devices = this[game][0].Machine.Devices; foreach (string device in devices) { // If the device doesn't exist then we continue if (this[device].Count == 0) { continue; } // Otherwise, copy the items from the device to the current game Machine musheen = this[game][0].Machine; List devItems = this[device]; foreach (DatItem item in devItems) { // Figure out the type of the item and add it accordingly switch (item.Type) { case ItemType.Archive: Archive archive = ((Archive)item).Clone() as Archive; archive.Machine = musheen; if (this[game].Where(i => i.Name == archive.Name).Count() == 0 && !this[game].Contains(archive)) { this[game].Add(archive); } break; case ItemType.BiosSet: BiosSet biosSet = ((BiosSet)item).Clone() as BiosSet; biosSet.Machine = musheen; if (this[game].Where(i => i.Name == biosSet.Name).Count() == 0 && !this[game].Contains(biosSet)) { this[game].Add(biosSet); } break; case ItemType.Disk: Disk disk = ((Disk)item).Clone() as Disk; disk.Machine = musheen; if (this[game].Where(i => i.Name == disk.Name).Count() == 0 && !this[game].Contains(disk)) { this[game].Add(disk); } break; case ItemType.Release: Release release = ((Release)item).Clone() as Release; release.Machine = musheen; if (this[game].Where(i => i.Name == release.Name).Count() == 0 && !this[game].Contains(release)) { this[game].Add(release); } break; case ItemType.Rom: Rom rom = ((Rom)item).Clone() as Rom; rom.Machine = musheen; if (this[game].Where(i => i.Name == rom.Name).Count() == 0 && !this[game].Contains(rom)) { this[game].Add(rom); } break; case ItemType.Sample: Sample sample = ((Sample)item).Clone() as Sample; sample.Machine = musheen; if (this[game].Where(i => i.Name == sample.Name).Count() == 0 && !this[game].Contains(sample)) { this[game].Add(sample); } break; } } } } } /// /// Use cloneof tags to add roms to the children, setting the new romof tag in the process /// private void AddRomsFromParent() { List games = Keys.ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue if (this[game].Count == 0) { continue; } // Determine if the game has a parent or not string parent = null; if (!String.IsNullOrEmpty(this[game][0].Machine.CloneOf)) { parent = this[game][0].Machine.CloneOf; } // If the parent doesnt exist, we want to continue if (String.IsNullOrEmpty(parent)) { continue; } // If the parent doesn't have any items, we want to continue if (this[parent].Count == 0) { continue; } // If the parent exists and has items, we copy the items from the parent to the current game Machine currentMachine = this[game][0].Machine; List parentItems = this[parent]; foreach (DatItem item in parentItems) { // Figure out the type of the item and add it accordingly switch (item.Type) { case ItemType.Archive: Archive archive = ((Archive)item).Clone() as Archive; archive.Machine = currentMachine; if (this[game].Where(i => i.Name == archive.Name).Count() == 0 && !this[game].Contains(archive)) { this[game].Add(archive); } break; case ItemType.BiosSet: BiosSet biosSet = ((BiosSet)item).Clone() as BiosSet; biosSet.Machine = currentMachine; if (this[game].Where(i => i.Name == biosSet.Name).Count() == 0 && !this[game].Contains(biosSet)) { this[game].Add(biosSet); } break; case ItemType.Disk: Disk disk = ((Disk)item).Clone() as Disk; disk.Machine = currentMachine; if (this[game].Where(i => i.Name == disk.Name).Count() == 0 && !this[game].Contains(disk)) { this[game].Add(disk); } break; case ItemType.Release: Release release = ((Release)item).Clone() as Release; release.Machine = currentMachine; if (this[game].Where(i => i.Name == release.Name).Count() == 0 && !this[game].Contains(release)) { this[game].Add(release); } break; case ItemType.Rom: Rom rom = ((Rom)item).Clone() as Rom; rom.Machine = currentMachine; if (this[game].Where(i => i.Name == rom.Name).Count() == 0 && !this[game].Contains(rom)) { this[game].Add(rom); } break; case ItemType.Sample: Sample sample = ((Sample)item).Clone() as Sample; sample.Machine = currentMachine; if (this[game].Where(i => i.Name == sample.Name).Count() == 0 && !this[game].Contains(sample)) { this[game].Add(sample); } break; } } // Now we want to get the parent romof tag and put it in each of the items List items = this[game]; string romof = this[parent][0].Machine.RomOf; foreach (DatItem item in items) { item.Machine.UpdateRomOf(romof); } } } /// /// Use cloneof tags to add roms to the parents, removing the child sets in the process /// private void AddRomsToParent() { List games = Keys.ToList(); foreach (string game in games) { // Determine if the game has a parent or not string parent = null; if (!String.IsNullOrEmpty(this[game][0].Machine.CloneOf)) { parent = this[game][0].Machine.CloneOf; } // If there is no parent, then we continue if (String.IsNullOrEmpty(parent)) { continue; } // Otherwise, move the items from the current game to a subfolder of the parent game Machine parentMachine = this[parent].Count == 0 ? new Machine { Name = parent, Description = parent } : this[parent][0].Machine; List items = this[game]; foreach (DatItem item in items) { // If the disk doesn't have a valid merge tag OR the merged file doesn't exist in the parent, then add it if (item.Type == ItemType.Disk && (item.MergeTag == null || !this[parent].Select(i => i.Name).Contains(item.MergeTag))) { item.Machine = parentMachine; this[parent].Add(item); } // Otherwise, if the parent doesn't already contain the non-disk, add it else if (item.Type != ItemType.Disk && !this[parent].Contains(item)) { // Rename the child so it's in a subfolder item.Name = item.Machine.Name + "\\" + item.Name; // Update the machine to be the new parent item.Machine = parentMachine; // Add the rom to the parent set this[parent].Add(item); } } // Then, remove the old game so it's not picked up by the writer Remove(game); } } /// /// Remove all BIOS and device sets /// private void RemoveBiosAndDeviceSets() { List games = Keys.ToList(); foreach (string game in games) { if (this[game].Count > 0 && (this[game][0].Machine.MachineType == MachineType.Bios || this[game][0].Machine.MachineType == MachineType.Device)) { Remove(game); } } } /// /// Use romof tags to remove roms from the children /// private void RemoveBiosRomsFromChild() { // Loop through the romof tags List games = Keys.ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue if (this[game].Count == 0) { continue; } // Determine if the game has a parent or not string parent = null; if (!String.IsNullOrEmpty(this[game][0].Machine.RomOf)) { parent = this[game][0].Machine.RomOf; } // If the parent doesnt exist, we want to continue if (String.IsNullOrEmpty(parent)) { continue; } // If the parent doesn't have any items, we want to continue if (this[parent].Count == 0) { continue; } // If the parent exists and has items, we remove the items that are in the parent from the current game Machine currentMachine = this[game][0].Machine; List parentItems = this[parent]; foreach (DatItem item in parentItems) { // Figure out the type of the item and add it accordingly switch (item.Type) { case ItemType.Archive: Archive archive = ((Archive)item).Clone() as Archive; if (this[game].Contains(archive)) { this[game].Remove(archive); } break; case ItemType.BiosSet: BiosSet biosSet = ((BiosSet)item).Clone() as BiosSet; if (this[game].Contains(biosSet)) { this[game].Remove(biosSet); } break; case ItemType.Disk: Disk disk = ((Disk)item).Clone() as Disk; if (this[game].Contains(disk)) { this[game].Remove(disk); } break; case ItemType.Release: Release release = ((Release)item).Clone() as Release; if (this[game].Contains(release)) { this[game].Remove(release); } break; case ItemType.Rom: Rom rom = ((Rom)item).Clone() as Rom; if (this[game].Contains(rom)) { this[game].Remove(rom); } break; case ItemType.Sample: Sample sample = ((Sample)item).Clone() as Sample; if (this[game].Contains(sample)) { this[game].Remove(sample); } break; } } } } /// /// Use cloneof tags to remove roms from the children /// private void RemoveRomsFromChild() { List games = Keys.ToList(); foreach (string game in games) { // If the game has no items in it, we want to continue if (this[game].Count == 0) { continue; } // Determine if the game has a parent or not string parent = null; if (!String.IsNullOrEmpty(this[game][0].Machine.CloneOf)) { parent = this[game][0].Machine.CloneOf; } // If the parent doesnt exist, we want to continue if (String.IsNullOrEmpty(parent)) { continue; } // If the parent doesn't have any items, we want to continue if (this[parent].Count == 0) { continue; } // If the parent exists and has items, we copy the items from the parent to the current game Machine currentMachine = this[game][0].Machine; List parentItems = this[parent]; foreach (DatItem item in parentItems) { // Figure out the type of the item and remove it accordingly switch (item.Type) { case ItemType.Archive: Archive archive = ((Archive)item).Clone() as Archive; if (this[game].Contains(archive)) { this[game].Remove(archive); } break; case ItemType.BiosSet: BiosSet biosSet = ((BiosSet)item).Clone() as BiosSet; if (this[game].Contains(biosSet)) { this[game].Remove(biosSet); } break; case ItemType.Disk: Disk disk = ((Disk)item).Clone() as Disk; if (this[game].Contains(disk)) { this[game].Remove(disk); } break; case ItemType.Release: Release release = ((Release)item).Clone() as Release; if (this[game].Contains(release)) { this[game].Remove(release); } break; case ItemType.Rom: Rom rom = ((Rom)item).Clone() as Rom; if (this[game].Contains(rom)) { this[game].Remove(rom); } break; case ItemType.Sample: Sample sample = ((Sample)item).Clone() as Sample; if (this[game].Contains(sample)) { this[game].Remove(sample); } break; } } // Now we want to get the parent romof tag and put it in each of the items List items = this[game]; string romof = this[parent][0].Machine.RomOf; foreach (DatItem item in items) { item.Machine.UpdateRomOf(romof); } } } /// /// Remove all romof and cloneof tags from all games /// private void RemoveTagsFromChild() { List games = Keys.ToList(); foreach (string game in games) { List items = this[game]; foreach (DatItem item in items) { item.Machine.UpdateCloneOf(null); item.Machine.UpdateRomOf(null); } } } #endregion #endregion // Instance Methods } }