diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index b2f4c600..2ee65f81 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using System.Text.RegularExpressions; using System.Threading.Tasks; using SabreTools.Library.Data; @@ -296,7 +298,7 @@ namespace SabreTools.Library.DatFiles // First we parse in the DAT internally DatFile intDat = Create(Header.CloneFiltering()); intDat.Parse(path, 1, keep: true); - filter.FilterDatFile(intDat, false /* useTags */); + intDat.ApplyFilter(filter, false /* useTags */); // If we are matching based on DatItem fields of any sort if (updateFields.Intersect(datItemFields).Any()) @@ -404,7 +406,7 @@ namespace SabreTools.Library.DatFiles // First we parse in the DAT internally DatFile intDat = Create(Header.CloneFiltering()); intDat.Parse(path, 1, keep: true); - filter.FilterDatFile(intDat, false /* useTags */); + intDat.ApplyFilter(filter, false /* useTags */); // For comparison's sake, we want to a the base bucketing if (useGames) @@ -896,7 +898,7 @@ namespace SabreTools.Library.DatFiles } // Now that we have a merged DAT, filter it - filter.FilterDatFile(this, false /* useTags */); + ApplyFilter(filter, false /* useTags */); watch.Stop(); @@ -934,7 +936,7 @@ namespace SabreTools.Library.DatFiles keepext: innerDatdata.Header.DatFormat.HasFlag(DatFormat.TSV) || innerDatdata.Header.DatFormat.HasFlag(DatFormat.CSV) || innerDatdata.Header.DatFormat.HasFlag(DatFormat.SSV)); - filter.FilterDatFile(innerDatdata, false /* useTags */); + innerDatdata.ApplyFilter(filter, false /* useTags */); // Get the correct output path string realOutDir = file.GetOutputPath(outDir, inplace); @@ -946,6 +948,1101 @@ namespace SabreTools.Library.DatFiles #endregion + #region Filtering + + /// + /// Apply a Filter on the DatFile + /// + /// Filter to use + /// True if DatFile tags override splitting, false otherwise + /// True if the DatFile was filtered, false on error + public bool ApplyFilter(Filter filter, bool useTags) + { + try + { + // Process description to machine name + if (filter.DescriptionAsName) + MachineDescriptionToName(); + + // If we are using tags from the DAT, set the proper input for split type unless overridden + if (useTags && filter.InternalSplit == SplitType.None) + filter.InternalSplit = Header.ForceMerging.AsSplitType(); + + // Run internal splitting + ProcessSplitType(filter.InternalSplit); + + // Loop over every key in the dictionary + List keys = Items.Keys.ToList(); + foreach (string key in keys) + { + // For every item in the current key + List items = Items[key]; + List newitems = new List(); + foreach (DatItem item in items) + { + // If we have a null item, we can't pass it + if (item == null) + continue; + + // If the rom passes the filter, include it + if (item.PassesFilter(filter)) + { + // If we're stripping unicode characters, do so from all relevant things + if (filter.RemoveUnicode) + { + item.Name = Sanitizer.RemoveUnicodeCharacters(item.Name); + item.MachineName = Sanitizer.RemoveUnicodeCharacters(item.MachineName); + item.MachineDescription = Sanitizer.RemoveUnicodeCharacters(item.MachineDescription); + } + + // If we're in cleaning mode, do so from all relevant things + if (filter.Clean) + { + item.MachineName = Sanitizer.CleanGameName(item.MachineName); + item.MachineDescription = Sanitizer.CleanGameName(item.MachineDescription); + } + + // If we are in single game mode, rename all games + if (filter.Single) + item.MachineName = "!"; + + // If we are in NTFS trim mode, trim the game name + if (filter.Trim) + { + // Windows max name length is 260 + int usableLength = 260 - item.MachineName.Length - filter.Root.Length; + if (item.Name.Length > usableLength) + { + string ext = Path.GetExtension(item.Name); + item.Name = item.Name.Substring(0, usableLength - ext.Length); + item.Name += ext; + } + } + + // Add the item to the output + newitems.Add(item); + } + } + + Items.Remove(key); + Items.AddRange(key, newitems); + } + + // If we are removing scene dates, do that now + if (Header.SceneDateStrip) + StripSceneDatesFromItems(); + + // Run the one rom per game logic, if required + if (Header.OneGamePerRegion) + OneGamePerRegion(); + + // Run the one rom per game logic, if required + if (Header.OneRomPerGame) + OneRomPerGame(); + + // If we are removing fields, do that now + if (filter.RemoveFields) + RemoveFieldsFromItems(); + + // We remove any blanks, if we aren't supposed to have any + if (!Header.KeepEmptyGames) + { + List possiblyEmptyKeys = Items.Keys.ToList(); + foreach (string key in possiblyEmptyKeys) + { + List items = Items[key]; + if (items == null || items.Count == 0) + { + Items.Remove(key); + continue; + } + + List newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList(); + + Items.Remove(key); + Items.AddRange(key, newitems); + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof + /// + public void MachineDescriptionToName() + { + try + { + // First we want to get a mapping for all games to description + ConcurrentDictionary mapping = new ConcurrentDictionary(); + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + { + List items = Items[key]; + foreach (DatItem item in items) + { + // If the key mapping doesn't exist, add it + mapping.TryAdd(item.MachineName, item.MachineDescription.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); + } + }); + + // Now we loop through every item and update accordingly + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + { + List items = Items[key]; + List newItems = new List(); + foreach (DatItem item in items) + { + // Update machine name + if (!string.IsNullOrWhiteSpace(item.MachineName) && mapping.ContainsKey(item.MachineName)) + item.MachineName = mapping[item.MachineName]; + + // Update cloneof + if (!string.IsNullOrWhiteSpace(item.CloneOf) && mapping.ContainsKey(item.CloneOf)) + item.CloneOf = mapping[item.CloneOf]; + + // Update romof + if (!string.IsNullOrWhiteSpace(item.RomOf) && mapping.ContainsKey(item.RomOf)) + item.RomOf = mapping[item.RomOf]; + + // Update sampleof + if (!string.IsNullOrWhiteSpace(item.SampleOf) && mapping.ContainsKey(item.SampleOf)) + item.SampleOf = mapping[item.SampleOf]; + + // Add the new item to the output list + newItems.Add(item); + } + + // Replace the old list of roms with the new one + Items.Remove(key); + Items.AddRange(key, newItems); + }); + } + catch (Exception ex) + { + Globals.Logger.Warning(ex.ToString()); + } + } + + /// + /// Filter a DAT using 1G1R logic given an ordered set of regions + /// + /// + /// In the most technical sense, the way that the region list is being used does not + /// confine its values to be just regions. Since it's essentially acting like a + /// specialized version of the machine name filter, anything that is usually encapsulated + /// in parenthesis would be matched on, including disc numbers, languages, editions, + /// and anything else commonly used. Please note that, unlike other existing 1G1R + /// solutions, this does not have the ability to contain custom mappings of parent + /// to clone sets based on name, nor does it have the ability to match on the + /// Release DatItem type. + /// + public void OneGamePerRegion() + { + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); + + // Then we want to get a mapping of all machines to parents + Dictionary> parents = new Dictionary>(); + foreach (string key in Items.Keys) + { + DatItem item = Items[key][0]; + + // Match on CloneOf first + if (!string.IsNullOrEmpty(item.CloneOf)) + { + if (!parents.ContainsKey(item.CloneOf.ToLowerInvariant())) + parents.Add(item.CloneOf.ToLowerInvariant(), new List()); + + parents[item.CloneOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); + } + + // Then by RomOf + else if (!string.IsNullOrEmpty(item.RomOf)) + { + if (!parents.ContainsKey(item.RomOf.ToLowerInvariant())) + parents.Add(item.RomOf.ToLowerInvariant(), new List()); + + parents[item.RomOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); + } + + // Otherwise, treat it as a parent + else + { + if (!parents.ContainsKey(item.MachineName.ToLowerInvariant())) + parents.Add(item.MachineName.ToLowerInvariant(), new List()); + + parents[item.MachineName.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); + } + } + + // If we have null region list, make it empty + List regions = Header.RegionList; + if (regions == null) + regions = new List(); + + // Once we have the full list of mappings, filter out games to keep + foreach (string key in parents.Keys) + { + // Find the first machine that matches the regions in order, if possible + string machine = default; + foreach (string region in regions) + { + machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase)); + if (machine != default) + break; + } + + // If we didn't get a match, use the parent + if (machine == default) + machine = key; + + // Remove the key from the list + parents[key].Remove(machine); + + // Remove the rest of the items from this key + parents[key].ForEach(k => Items.Remove(k)); + } + + // Finally, strip out the parent tags + RemoveTagsFromChild(); + } + + /// + /// Ensure that all roms are in their own game (or at least try to ensure) + /// + public void OneRomPerGame() + { + // Because this introduces subfolders, we need to set the SuperDAT type + Header.Type = "SuperDAT"; + + // For each rom, we want to update the game to be "/" + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + { + List items = Items[key]; + for (int i = 0; i < items.Count; i++) + { + string[] splitname = items[i].Name.Split('.'); + items[i].MachineName += $"/{string.Join(".", splitname.Take(splitname.Length > 1 ? splitname.Length - 1 : 1))}"; + items[i].Name = Path.GetFileName(items[i].Name); + } + }); + } + + /// + /// Remove fields as per the header + /// + /// TODO: Move internal logic to DatItem like the rest + public void RemoveFieldsFromItems() + { + // Output the logging statement + Globals.Logger.User("Removing filtered fields"); + + // Get the array of fields from the header + List fields = Header.ExcludeFields; + + // Now process all of the roms + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + { + List items = Items[key]; + for (int j = 0; j < items.Count; j++) + { + DatItem item = items[j]; + + // TODO: Switch statement + foreach (Field field in fields) + { + // Machine Fields + if (field == Field.MachineName) + item.MachineName = null; + if (field == Field.Comment) + item.Comment = null; + if (field == Field.Description) + item.MachineDescription = null; + if (field == Field.Year) + item.Year = null; + if (field == Field.Manufacturer) + item.Manufacturer = null; + if (field == Field.Publisher) + item.Publisher = null; + if (field == Field.Category) + item.Category = null; + if (field == Field.RomOf) + item.RomOf = null; + if (field == Field.CloneOf) + item.CloneOf = null; + if (field == Field.SampleOf) + item.SampleOf = null; + if (field == Field.Supported) + item.Supported = null; + if (field == Field.SourceFile) + item.SourceFile = null; + if (field == Field.Runnable) + item.Runnable = null; + if (field == Field.Board) + item.Board = null; + if (field == Field.RebuildTo) + item.RebuildTo = null; + if (field == Field.Devices) + item.Devices = null; + if (field == Field.SlotOptions) + item.SlotOptions = null; + if (field == Field.Infos) + item.Infos = null; + if (field == Field.MachineType) + item.MachineType = MachineType.NULL; + + // Item Fields + if (field == Field.Name) + item.Name = null; + if (field == Field.PartName) + item.PartName = null; + if (field == Field.PartInterface) + item.PartInterface = null; + if (field == Field.Features) + item.Features = null; + if (field == Field.AreaName) + item.AreaName = null; + if (field == Field.AreaSize) + item.AreaSize = null; + if (field == Field.Default) + { + if (item.ItemType == ItemType.BiosSet) + (item as BiosSet).Default = null; + else if (item.ItemType == ItemType.Release) + (item as Release).Default = null; + } + if (field == Field.BiosDescription) + { + if (item.ItemType == ItemType.BiosSet) + (item as BiosSet).Description = null; + } + if (field == Field.Size) + { + if (item.ItemType == ItemType.Rom) + (item as Rom).Size = 0; + } + if (field == Field.CRC) + { + if (item.ItemType == ItemType.Rom) + (item as Rom).CRC = null; + } + if (field == Field.MD5) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).MD5 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).MD5 = null; + } +#if NET_FRAMEWORK + if (field == Field.RIPEMD160) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).RIPEMD160 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).RIPEMD160 = null; + } +#endif + if (field == Field.SHA1) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).SHA1 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).SHA1 = null; + } + if (field == Field.SHA256) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).SHA256 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).SHA256 = null; + } + if (field == Field.SHA384) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).SHA384 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).SHA384 = null; + } + if (field == Field.SHA512) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).SHA512 = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).SHA512 = null; + } + if (field == Field.Merge) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).MergeTag = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).MergeTag = null; + } + if (field == Field.Region) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).Region = null; + else if (item.ItemType == ItemType.Release) + (item as Release).Region = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).Region = null; + } + if (field == Field.Index) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).Index = null; + } + if (field == Field.Writable) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).Writable = null; + } + if (field == Field.Optional) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).Optional = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).Optional = null; + } + if (field == Field.Status) + { + if (item.ItemType == ItemType.Disk) + (item as Disk).ItemStatus = ItemStatus.NULL; + else if (item.ItemType == ItemType.Rom) + (item as Rom).ItemStatus = ItemStatus.NULL; + } + if (field == Field.Language) + { + if (item.ItemType == ItemType.Release) + (item as Release).Language = null; + } + if (field == Field.Date) + { + if (item.ItemType == ItemType.Release) + (item as Release).Date = null; + else if (item.ItemType == ItemType.Rom) + (item as Rom).Date = null; + } + if (field == Field.Bios) + { + if (item.ItemType == ItemType.Rom) + (item as Rom).Bios = null; + } + if (field == Field.Offset) + { + if (item.ItemType == ItemType.Rom) + (item as Rom).Offset = null; + } + if (field == Field.Inverted) + { + if (item.ItemType == ItemType.Rom) + (item as Rom).Inverted = null; + } + } + + items[j] = item; + } + + Items.Remove(key); + Items.AddRange(key, items); + }); + } + + /// + /// Strip the dates from the beginning of scene-style set names + /// + public void StripSceneDatesFromItems() + { + // Output the logging statement + Globals.Logger.User("Stripping scene-style dates"); + + // Set the regex pattern to use + string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)"; + + // Now process all of the roms + Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => + { + List items = Items[key]; + for (int j = 0; j < items.Count; j++) + { + DatItem item = items[j]; + if (Regex.IsMatch(item.MachineName, pattern)) + item.MachineName = Regex.Replace(item.MachineName, pattern, "$2"); + + if (Regex.IsMatch(item.MachineDescription, pattern)) + item.MachineDescription = Regex.Replace(item.MachineDescription, pattern, "$2"); + + items[j] = item; + } + + Items.Remove(key); + Items.AddRange(key, items); + }); + } + + #endregion + + #region Internal Splitting/Merging + + /// + /// Process items according to SplitType + /// + /// SplitType to implement + public void ProcessSplitType(SplitType splitType) + { + // Now we pre-process the DAT with the splitting/merging mode + switch (splitType) + { + case SplitType.None: + // No-op + break; + case SplitType.DeviceNonMerged: + CreateDeviceNonMergedSets(DedupeType.None); + break; + case SplitType.FullNonMerged: + CreateFullyNonMergedSets(DedupeType.None); + break; + case SplitType.NonMerged: + CreateNonMergedSets(DedupeType.None); + break; + case SplitType.Merged: + CreateMergedSets(DedupeType.None); + break; + case SplitType.Split: + CreateSplitSets(DedupeType.None); + break; + } + } + + /// + /// Use cdevice_ref tags to get full non-merged sets and remove parenting tags + /// + /// Dedupe type to be used + private void CreateDeviceNonMergedSets(DedupeType mergeroms) + { + Globals.Logger.User("Creating device non-merged sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); + + // Now we want to loop through all of the games and set the correct information + while (AddRomsFromDevices(false, false)) ; + while (AddRomsFromDevices(true, false)) ; + + // Then, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(); + } + + /// + /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets + /// + /// Dedupe type to be used + private void CreateFullyNonMergedSets(DedupeType mergeroms) + { + Globals.Logger.User("Creating fully non-merged sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); + + // Now we want to loop through all of the games and set the correct information + while (AddRomsFromDevices(true, true)) ; + AddRomsFromDevices(false, true); + AddRomsFromParent(); + + // Now that we have looped through the cloneof tags, we loop through the romof tags + AddRomsFromBios(); + + // Then, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(); + } + + /// + /// Use cloneof tags to create merged sets and remove the tags + /// + /// Dedupe type to be used + private void CreateMergedSets(DedupeType mergeroms) + { + Globals.Logger.User("Creating merged sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); + + // Now we want to loop through all of the games and set the correct information + AddRomsFromChildren(); + + // Now that we have looped through the cloneof tags, we loop through the romof tags + RemoveBiosRomsFromChild(false); + RemoveBiosRomsFromChild(true); + + // Finally, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(); + } + + /// + /// Use cloneof tags to create non-merged sets and remove the tags + /// + /// Dedupe type to be used + private void CreateNonMergedSets(DedupeType mergeroms) + { + Globals.Logger.User("Creating non-merged sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); + + // Now we want to loop through all of the games and set the correct information + AddRomsFromParent(); + + // Now that we have looped through the cloneof tags, we loop through the romof tags + RemoveBiosRomsFromChild(false); + RemoveBiosRomsFromChild(true); + + // Finally, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(); + } + + /// + /// Use cloneof and romof tags to create split sets and remove the tags + /// + /// Dedupe type to be used + private void CreateSplitSets(DedupeType mergeroms) + { + Globals.Logger.User("Creating split sets from the DAT"); + + // For sake of ease, the first thing we want to do is bucket by game + Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); + + // Now we want to loop through all of the games and set the correct information + RemoveRomsFromChild(); + + // Now that we have looped through the cloneof tags, we loop through the romof tags + RemoveBiosRomsFromChild(false); + RemoveBiosRomsFromChild(true); + + // Finally, remove the romof and cloneof tags so it's not picked up by the manager + RemoveTagsFromChild(); + } + + /// + /// Use romof tags to add roms to the children + /// + private void AddRomsFromBios() + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game has no items in it, we want to continue + if (Items[game].Count == 0) + continue; + + // Determine if the game has a parent or not + string parent = null; + if (!string.IsNullOrWhiteSpace(Items[game][0].RomOf)) + parent = Items[game][0].RomOf; + + // If the parent doesnt exist, we want to continue + if (string.IsNullOrWhiteSpace(parent)) + continue; + + // If the parent doesn't have any items, we want to continue + if (Items[parent].Count == 0) + continue; + + // If the parent exists and has items, we copy the items from the parent to the current game + DatItem copyFrom = Items[game][0]; + List parentItems = Items[parent]; + foreach (DatItem item in parentItems) + { + DatItem datItem = (DatItem)item.Clone(); + datItem.CopyMachineInformation(copyFrom); + if (Items[game].Where(i => i.Name == datItem.Name).Count() == 0 && !Items[game].Contains(datItem)) + Items.Add(game, datItem); + } + } + } + + /// + /// Use device_ref and optionally slotoption tags to add roms to the children + /// + /// True if only child device sets are touched, false for non-device sets (default) + /// True if slotoptions tags are used as well, false otherwise + private bool AddRomsFromDevices(bool dev = false, bool slotoptions = false) + { + bool foundnew = false; + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game doesn't have items, we continue + if (Items[game] == null || Items[game].Count == 0) + continue; + + // If the game (is/is not) a bios, we want to continue + if (dev ^ (Items[game][0].MachineType.HasFlag(MachineType.Device))) + continue; + + // If the game has no devices, we continue + if (Items[game][0].Devices == null + || Items[game][0].Devices.Count == 0 + || (slotoptions && Items[game][0].SlotOptions == null) + || (slotoptions && Items[game][0].SlotOptions.Count == 0)) + { + continue; + } + + // Determine if the game has any devices or not + List devices = Items[game][0].Devices; + List newdevs = new List(); + foreach (string device in devices) + { + // If the device doesn't exist then we continue + if (Items[device].Count == 0) + continue; + + // Otherwise, copy the items from the device to the current game + DatItem copyFrom = Items[game][0]; + List devItems = Items[device]; + foreach (DatItem item in devItems) + { + DatItem datItem = (DatItem)item.Clone(); + newdevs.AddRange(datItem.Devices ?? new List()); + datItem.CopyMachineInformation(copyFrom); + if (Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) + { + foundnew = true; + Items.Add(game, datItem); + } + } + } + + // Now that every device is accounted for, add the new list of devices, if they don't already exist + foreach (string device in newdevs) + { + if (!Items[game][0].Devices.Contains(device)) + Items[game][0].Devices.Add(device); + } + + // If we're checking slotoptions too + if (slotoptions) + { + // Determine if the game has any slotoptions or not + List slotopts = Items[game][0].SlotOptions; + List newslotopts = new List(); + foreach (string slotopt in slotopts) + { + // If the slotoption doesn't exist then we continue + if (Items[slotopt].Count == 0) + continue; + + // Otherwise, copy the items from the slotoption to the current game + DatItem copyFrom = Items[game][0]; + List slotItems = Items[slotopt]; + foreach (DatItem item in slotItems) + { + DatItem datItem = (DatItem)item.Clone(); + newslotopts.AddRange(datItem.SlotOptions ?? new List()); + datItem.CopyMachineInformation(copyFrom); + if (Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) + { + foundnew = true; + Items.Add(game, datItem); + } + } + } + + // Now that every slotoption is accounted for, add the new list of slotoptions, if they don't already exist + foreach (string slotopt in newslotopts) + { + if (!Items[game][0].SlotOptions.Contains(slotopt)) + Items[game][0].SlotOptions.Add(slotopt); + } + } + } + + return foundnew; + } + + /// + /// Use cloneof tags to add roms to the children, setting the new romof tag in the process + /// + private void AddRomsFromParent() + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game has no items in it, we want to continue + if (Items[game].Count == 0) + continue; + + // Determine if the game has a parent or not + string parent = null; + if (!string.IsNullOrWhiteSpace(Items[game][0].CloneOf)) + parent = Items[game][0].CloneOf; + + // If the parent doesnt exist, we want to continue + if (string.IsNullOrWhiteSpace(parent)) + continue; + + // If the parent doesn't have any items, we want to continue + if (Items[parent].Count == 0) + continue; + + // If the parent exists and has items, we copy the items from the parent to the current game + DatItem copyFrom = Items[game][0]; + List parentItems = Items[parent]; + foreach (DatItem item in parentItems) + { + DatItem datItem = (DatItem)item.Clone(); + datItem.CopyMachineInformation(copyFrom); + if (Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0 + && !Items[game].Contains(datItem)) + { + Items.Add(game, datItem); + } + } + + // Now we want to get the parent romof tag and put it in each of the items + List items = Items[game]; + string romof = Items[parent][0].RomOf; + foreach (DatItem item in items) + { + item.RomOf = romof; + } + } + } + + /// + /// Use cloneof tags to add roms to the parents, removing the child sets in the process + /// + /// True to add DatItems to subfolder of parent (not including Disk), false otherwise + private void AddRomsFromChildren(bool subfolder = true) + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game has no items in it, we want to continue + if (Items[game].Count == 0) + continue; + + // Determine if the game has a parent or not + string parent = null; + if (!string.IsNullOrWhiteSpace(Items[game][0].CloneOf)) + parent = Items[game][0].CloneOf; + + // If there is no parent, then we continue + if (string.IsNullOrWhiteSpace(parent)) + continue; + + // Otherwise, move the items from the current game to a subfolder of the parent game + DatItem copyFrom = Items[parent].Count == 0 ? new Rom { MachineName = parent, MachineDescription = parent } : Items[parent][0]; + List items = Items[game]; + foreach (DatItem item in items) + { + // Special disk handling + if (item.ItemType == ItemType.Disk) + { + Disk disk = item as Disk; + + // If the merge tag exists and the parent already contains it, skip + if (disk.MergeTag != null && Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) + { + continue; + } + + // If the merge tag exists but the parent doesn't contain it, add to parent + else if (disk.MergeTag != null && !Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) + { + item.CopyMachineInformation(copyFrom); + Items.Add(parent, item); + } + + // If there is no merge tag, add to parent + else if (disk.MergeTag == null) + { + item.CopyMachineInformation(copyFrom); + Items.Add(parent, item); + } + } + + // Special rom handling + else if (item.ItemType == ItemType.Rom) + { + Rom rom = item as Rom; + + // If the merge tag exists and the parent already contains it, skip + if (rom.MergeTag != null && Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) + { + continue; + } + + // If the merge tag exists but the parent doesn't contain it, add to subfolder of parent + else if (rom.MergeTag != null && !Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) + { + if (subfolder) + item.Name = $"{item.MachineName}\\{item.Name}"; + + item.CopyMachineInformation(copyFrom); + Items.Add(parent, item); + } + + // If the parent doesn't already contain this item, add to subfolder of parent + else if (!Items[parent].Contains(item)) + { + if (subfolder) + item.Name = $"{item.MachineName}\\{item.Name}"; + + item.CopyMachineInformation(copyFrom); + Items.Add(parent, item); + } + } + + // All other that would be missing to subfolder of parent + else if (!Items[parent].Contains(item)) + { + if (subfolder) + item.Name = $"{item.MachineName}\\{item.Name}"; + + item.CopyMachineInformation(copyFrom); + Items.Add(parent, item); + } + } + + // Then, remove the old game so it's not picked up by the writer + Items.Remove(game); + } + } + + /// + /// Remove all BIOS and device sets + /// + private void RemoveBiosAndDeviceSets() + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + if (Items[game].Count > 0 + && (Items[game][0].MachineType.HasFlag(MachineType.Bios) + || Items[game][0].MachineType.HasFlag(MachineType.Device))) + { + Items.Remove(game); + } + } + } + + /// + /// Use romof tags to remove bios roms from children + /// + /// True if only child Bios sets are touched, false for non-bios sets (default) + private void RemoveBiosRomsFromChild(bool bios = false) + { + // Loop through the romof tags + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game has no items in it, we want to continue + if (Items[game].Count == 0) + continue; + + // If the game (is/is not) a bios, we want to continue + if (bios ^ Items[game][0].MachineType.HasFlag(MachineType.Bios)) + continue; + + // Determine if the game has a parent or not + string parent = null; + if (!string.IsNullOrWhiteSpace(Items[game][0].RomOf)) + parent = Items[game][0].RomOf; + + // If the parent doesnt exist, we want to continue + if (string.IsNullOrWhiteSpace(parent)) + continue; + + // If the parent doesn't have any items, we want to continue + if (Items[parent].Count == 0) + continue; + + // If the parent exists and has items, we remove the items that are in the parent from the current game + List parentItems = Items[parent]; + foreach (DatItem item in parentItems) + { + DatItem datItem = (DatItem)item.Clone(); + while (Items[game].Contains(datItem)) + { + Items.Remove(game, datItem); + } + } + } + } + + /// + /// Use cloneof tags to remove roms from the children + /// + private void RemoveRomsFromChild() + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + // If the game has no items in it, we want to continue + if (Items[game].Count == 0) + continue; + + // Determine if the game has a parent or not + string parent = null; + if (!string.IsNullOrWhiteSpace(Items[game][0].CloneOf)) + parent = Items[game][0].CloneOf; + + // If the parent doesnt exist, we want to continue + if (string.IsNullOrWhiteSpace(parent)) + continue; + + // If the parent doesn't have any items, we want to continue + if (Items[parent].Count == 0) + continue; + + // If the parent exists and has items, we remove the parent items from the current game + List parentItems = Items[parent]; + foreach (DatItem item in parentItems) + { + DatItem datItem = (DatItem)item.Clone(); + while (Items[game].Contains(datItem)) + { + Items.Remove(game, datItem); + } + } + + // Now we want to get the parent romof tag and put it in each of the remaining items + List items = Items[game]; + string romof = Items[parent][0].RomOf; + foreach (DatItem item in items) + { + item.RomOf = romof; + } + } + } + + /// + /// Remove all romof and cloneof tags from all games + /// + private void RemoveTagsFromChild() + { + List games = Items.Keys.OrderBy(g => g).ToList(); + foreach (string game in games) + { + List items = Items[game]; + foreach (DatItem item in items) + { + item.CloneOf = null; + item.RomOf = null; + item.SampleOf = null; + } + } + } + + #endregion + #region Parsing /// @@ -1248,7 +2345,7 @@ namespace SabreTools.Library.DatFiles // If we have a valid filter, perform the filtering now if (filter != null && filter != default(Filter)) - filter.FilterDatFile(this, useTags); + ApplyFilter(filter, useTags); return true; } diff --git a/SabreTools.Library/Filtering/Filter.cs b/SabreTools.Library/Filtering/Filter.cs index 9da95b56..6b0ef8a1 100644 --- a/SabreTools.Library/Filtering/Filter.cs +++ b/SabreTools.Library/Filtering/Filter.cs @@ -1,11 +1,5 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - using SabreTools.Library.Data; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; @@ -794,1124 +788,6 @@ namespace SabreTools.Library.Filtering #endregion - #region Filter Running - - /// - /// Filter a DatFile using the inputs - /// - /// DatFile to filter - /// True if DatFile tags override splitting, false otherwise - /// True if the DatFile was filtered, false on error - /// TODO: Should this be inverted to be in DatFile? - public bool FilterDatFile(DatFile datFile, bool useTags) - { - try - { - // Process description to machine name - if (DescriptionAsName) - MachineDescriptionToName(datFile); - - // If we are using tags from the DAT, set the proper input for split type unless overridden - if (useTags && InternalSplit == SplitType.None) - InternalSplit = datFile.Header.ForceMerging.AsSplitType(); - - // Run internal splitting - ProcessSplitType(datFile, InternalSplit); - - // Loop over every key in the dictionary - List keys = datFile.Items.Keys.ToList(); - foreach (string key in keys) - { - // For every item in the current key - List items = datFile.Items[key]; - List newitems = new List(); - foreach (DatItem item in items) - { - // If we have a null item, we can't pass it - if (item == null) - continue; - - // If the rom passes the filter, include it - if (item.PassesFilter(this)) - { - // If we're stripping unicode characters, do so from all relevant things - if (RemoveUnicode) - { - item.Name = Sanitizer.RemoveUnicodeCharacters(item.Name); - item.MachineName = Sanitizer.RemoveUnicodeCharacters(item.MachineName); - item.MachineDescription = Sanitizer.RemoveUnicodeCharacters(item.MachineDescription); - } - - // If we're in cleaning mode, do so from all relevant things - if (Clean) - { - item.MachineName = Sanitizer.CleanGameName(item.MachineName); - item.MachineDescription = Sanitizer.CleanGameName(item.MachineDescription); - } - - // If we are in single game mode, rename all games - if (Single) - item.MachineName = "!"; - - // If we are in NTFS trim mode, trim the game name - if (Trim) - { - // Windows max name length is 260 - int usableLength = 260 - item.MachineName.Length - Root.Length; - if (item.Name.Length > usableLength) - { - string ext = Path.GetExtension(item.Name); - item.Name = item.Name.Substring(0, usableLength - ext.Length); - item.Name += ext; - } - } - - // Add the item to the output - newitems.Add(item); - } - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newitems); - } - - // If we are removing scene dates, do that now - if (datFile.Header.SceneDateStrip) - StripSceneDatesFromItems(datFile); - - // Run the one rom per game logic, if required - if (datFile.Header.OneGamePerRegion) - OneGamePerRegion(datFile); - - // Run the one rom per game logic, if required - if (datFile.Header.OneRomPerGame) - OneRomPerGame(datFile); - - // If we are removing fields, do that now - if (RemoveFields) - RemoveFieldsFromItems(datFile); - - // We remove any blanks, if we aren't supposed to have any - if (!datFile.Header.KeepEmptyGames) - { - List possiblyEmptyKeys = datFile.Items.Keys.ToList(); - foreach (string key in possiblyEmptyKeys) - { - List items = datFile.Items[key]; - if (items == null || items.Count == 0) - { - datFile.Items.Remove(key); - continue; - } - - List newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList(); - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newitems); - } - } - } - catch (Exception ex) - { - Globals.Logger.Error(ex.ToString()); - return false; - } - - return true; - } - - #endregion - - #region Internal Splitting/Merging - - /// - /// Process items according to SplitType - /// - /// DatFile to filter - /// SplitType to implement - private void ProcessSplitType(DatFile datFile, SplitType splitType) - { - // Now we pre-process the DAT with the splitting/merging mode - switch (splitType) - { - case SplitType.None: - // No-op - break; - case SplitType.DeviceNonMerged: - CreateDeviceNonMergedSets(datFile, DedupeType.None); - break; - case SplitType.FullNonMerged: - CreateFullyNonMergedSets(datFile, DedupeType.None); - break; - case SplitType.NonMerged: - CreateNonMergedSets(datFile, DedupeType.None); - break; - case SplitType.Merged: - CreateMergedSets(datFile, DedupeType.None); - break; - case SplitType.Split: - CreateSplitSets(datFile, DedupeType.None); - break; - } - } - - /// - /// Use cdevice_ref tags to get full non-merged sets and remove parenting tags - /// - /// DatFile to filter - /// Dedupe type to be used - private void CreateDeviceNonMergedSets(DatFile datFile, DedupeType mergeroms) - { - Globals.Logger.User("Creating device non-merged sets from the DAT"); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); - - // Now we want to loop through all of the games and set the correct information - while (AddRomsFromDevices(datFile, false, false)) ; - while (AddRomsFromDevices(datFile, true, false)) ; - - // Then, remove the romof and cloneof tags so it's not picked up by the manager - RemoveTagsFromChild(datFile); - } - - /// - /// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets - /// - /// DatFile to filter - /// Dedupe type to be used - private void CreateFullyNonMergedSets(DatFile datFile, DedupeType mergeroms) - { - Globals.Logger.User("Creating fully non-merged sets from the DAT"); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); - - // Now we want to loop through all of the games and set the correct information - while (AddRomsFromDevices(datFile, true, true)) ; - AddRomsFromDevices(datFile, false, true); - AddRomsFromParent(datFile); - - // Now that we have looped through the cloneof tags, we loop through the romof tags - AddRomsFromBios(datFile); - - // Then, remove the romof and cloneof tags so it's not picked up by the manager - RemoveTagsFromChild(datFile); - } - - /// - /// Use cloneof tags to create merged sets and remove the tags - /// - /// DatFile to filter - /// Dedupe type to be used - private void CreateMergedSets(DatFile datFile, DedupeType mergeroms) - { - Globals.Logger.User("Creating merged sets from the DAT"); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); - - // Now we want to loop through all of the games and set the correct information - AddRomsFromChildren(datFile); - - // Now that we have looped through the cloneof tags, we loop through the romof tags - RemoveBiosRomsFromChild(datFile, false); - RemoveBiosRomsFromChild(datFile, true); - - // Finally, remove the romof and cloneof tags so it's not picked up by the manager - RemoveTagsFromChild(datFile); - } - - /// - /// Use cloneof tags to create non-merged sets and remove the tags - /// - /// DatFile to filter - /// Dedupe type to be used - private void CreateNonMergedSets(DatFile datFile, DedupeType mergeroms) - { - Globals.Logger.User("Creating non-merged sets from the DAT"); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); - - // Now we want to loop through all of the games and set the correct information - AddRomsFromParent(datFile); - - // Now that we have looped through the cloneof tags, we loop through the romof tags - RemoveBiosRomsFromChild(datFile, false); - RemoveBiosRomsFromChild(datFile, true); - - // Finally, remove the romof and cloneof tags so it's not picked up by the manager - RemoveTagsFromChild(datFile); - } - - /// - /// Use cloneof and romof tags to create split sets and remove the tags - /// - /// DatFile to filter - /// Dedupe type to be used - private void CreateSplitSets(DatFile datFile, DedupeType mergeroms) - { - Globals.Logger.User("Creating split sets from the DAT"); - - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, mergeroms, norename: true); - - // Now we want to loop through all of the games and set the correct information - RemoveRomsFromChild(datFile); - - // Now that we have looped through the cloneof tags, we loop through the romof tags - RemoveBiosRomsFromChild(datFile, false); - RemoveBiosRomsFromChild(datFile, true); - - // Finally, remove the romof and cloneof tags so it's not picked up by the manager - RemoveTagsFromChild(datFile); - } - - /// - /// Use romof tags to add roms to the children - /// - /// DatFile to filter - private void AddRomsFromBios(DatFile datFile) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game has no items in it, we want to continue - if (datFile.Items[game].Count == 0) - continue; - - // Determine if the game has a parent or not - string parent = null; - if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].RomOf)) - parent = datFile.Items[game][0].RomOf; - - // If the parent doesnt exist, we want to continue - if (string.IsNullOrWhiteSpace(parent)) - continue; - - // If the parent doesn't have any items, we want to continue - if (datFile.Items[parent].Count == 0) - continue; - - // If the parent exists and has items, we copy the items from the parent to the current game - DatItem copyFrom = datFile.Items[game][0]; - List parentItems = datFile.Items[parent]; - foreach (DatItem item in parentItems) - { - DatItem datItem = (DatItem)item.Clone(); - datItem.CopyMachineInformation(copyFrom); - if (datFile.Items[game].Where(i => i.Name == datItem.Name).Count() == 0 && !datFile.Items[game].Contains(datItem)) - datFile.Items.Add(game, datItem); - } - } - } - - /// - /// Use device_ref and optionally slotoption tags to add roms to the children - /// - /// DatFile to filter - /// True if only child device sets are touched, false for non-device sets (default) - /// True if slotoptions tags are used as well, false otherwise - private bool AddRomsFromDevices(DatFile datFile, bool dev = false, bool slotoptions = false) - { - bool foundnew = false; - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game doesn't have items, we continue - if (datFile.Items[game] == null || datFile.Items[game].Count == 0) - continue; - - // If the game (is/is not) a bios, we want to continue - if (dev ^ (datFile.Items[game][0].MachineType.HasFlag(MachineType.Device))) - continue; - - // If the game has no devices, we continue - if (datFile.Items[game][0].Devices == null - || datFile.Items[game][0].Devices.Count == 0 - || (slotoptions && datFile.Items[game][0].SlotOptions == null) - || (slotoptions && datFile.Items[game][0].SlotOptions.Count == 0)) - { - continue; - } - - // Determine if the game has any devices or not - List devices = datFile.Items[game][0].Devices; - List newdevs = new List(); - foreach (string device in devices) - { - // If the device doesn't exist then we continue - if (datFile.Items[device].Count == 0) - continue; - - // Otherwise, copy the items from the device to the current game - DatItem copyFrom = datFile.Items[game][0]; - List devItems = datFile.Items[device]; - foreach (DatItem item in devItems) - { - DatItem datItem = (DatItem)item.Clone(); - newdevs.AddRange(datItem.Devices ?? new List()); - datItem.CopyMachineInformation(copyFrom); - if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) - { - foundnew = true; - datFile.Items.Add(game, datItem); - } - } - } - - // Now that every device is accounted for, add the new list of devices, if they don't already exist - foreach (string device in newdevs) - { - if (!datFile.Items[game][0].Devices.Contains(device)) - datFile.Items[game][0].Devices.Add(device); - } - - // If we're checking slotoptions too - if (slotoptions) - { - // Determine if the game has any slotoptions or not - List slotopts = datFile.Items[game][0].SlotOptions; - List newslotopts = new List(); - foreach (string slotopt in slotopts) - { - // If the slotoption doesn't exist then we continue - if (datFile.Items[slotopt].Count == 0) - continue; - - // Otherwise, copy the items from the slotoption to the current game - DatItem copyFrom = datFile.Items[game][0]; - List slotItems = datFile.Items[slotopt]; - foreach (DatItem item in slotItems) - { - DatItem datItem = (DatItem)item.Clone(); - newslotopts.AddRange(datItem.SlotOptions ?? new List()); - datItem.CopyMachineInformation(copyFrom); - if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0) - { - foundnew = true; - datFile.Items.Add(game, datItem); - } - } - } - - // Now that every slotoption is accounted for, add the new list of slotoptions, if they don't already exist - foreach (string slotopt in newslotopts) - { - if (!datFile.Items[game][0].SlotOptions.Contains(slotopt)) - datFile.Items[game][0].SlotOptions.Add(slotopt); - } - } - } - - return foundnew; - } - - /// - /// Use cloneof tags to add roms to the children, setting the new romof tag in the process - /// - /// DatFile to filter - private void AddRomsFromParent(DatFile datFile) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game has no items in it, we want to continue - if (datFile.Items[game].Count == 0) - continue; - - // Determine if the game has a parent or not - string parent = null; - if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) - parent = datFile.Items[game][0].CloneOf; - - // If the parent doesnt exist, we want to continue - if (string.IsNullOrWhiteSpace(parent)) - continue; - - // If the parent doesn't have any items, we want to continue - if (datFile.Items[parent].Count == 0) - continue; - - // If the parent exists and has items, we copy the items from the parent to the current game - DatItem copyFrom = datFile.Items[game][0]; - List parentItems = datFile.Items[parent]; - foreach (DatItem item in parentItems) - { - DatItem datItem = (DatItem)item.Clone(); - datItem.CopyMachineInformation(copyFrom); - if (datFile.Items[game].Where(i => i.Name.ToLowerInvariant() == datItem.Name.ToLowerInvariant()).Count() == 0 - && !datFile.Items[game].Contains(datItem)) - { - datFile.Items.Add(game, datItem); - } - } - - // Now we want to get the parent romof tag and put it in each of the items - List items = datFile.Items[game]; - string romof = datFile.Items[parent][0].RomOf; - foreach (DatItem item in items) - { - item.RomOf = romof; - } - } - } - - /// - /// Use cloneof tags to add roms to the parents, removing the child sets in the process - /// - /// DatFile to filter - /// True to add DatItems to subfolder of parent (not including Disk), false otherwise - private void AddRomsFromChildren(DatFile datFile, bool subfolder = true) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game has no items in it, we want to continue - if (datFile.Items[game].Count == 0) - continue; - - // Determine if the game has a parent or not - string parent = null; - if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) - parent = datFile.Items[game][0].CloneOf; - - // If there is no parent, then we continue - if (string.IsNullOrWhiteSpace(parent)) - continue; - - // Otherwise, move the items from the current game to a subfolder of the parent game - DatItem copyFrom = datFile.Items[parent].Count == 0 ? new Rom { MachineName = parent, MachineDescription = parent } : datFile.Items[parent][0]; - List items = datFile.Items[game]; - foreach (DatItem item in items) - { - // Special disk handling - if (item.ItemType == ItemType.Disk) - { - Disk disk = item as Disk; - - // If the merge tag exists and the parent already contains it, skip - if (disk.MergeTag != null && datFile.Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) - { - continue; - } - - // If the merge tag exists but the parent doesn't contain it, add to parent - else if (disk.MergeTag != null && !datFile.Items[parent].Select(i => i.Name).Contains(disk.MergeTag)) - { - item.CopyMachineInformation(copyFrom); - datFile.Items.Add(parent, item); - } - - // If there is no merge tag, add to parent - else if (disk.MergeTag == null) - { - item.CopyMachineInformation(copyFrom); - datFile.Items.Add(parent, item); - } - } - - // Special rom handling - else if (item.ItemType == ItemType.Rom) - { - Rom rom = item as Rom; - - // If the merge tag exists and the parent already contains it, skip - if (rom.MergeTag != null && datFile.Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) - { - continue; - } - - // If the merge tag exists but the parent doesn't contain it, add to subfolder of parent - else if (rom.MergeTag != null && !datFile.Items[parent].Select(i => i.Name).Contains(rom.MergeTag)) - { - if (subfolder) - item.Name = $"{item.MachineName}\\{item.Name}"; - - item.CopyMachineInformation(copyFrom); - datFile.Items.Add(parent, item); - } - - // If the parent doesn't already contain this item, add to subfolder of parent - else if (!datFile.Items[parent].Contains(item)) - { - if (subfolder) - item.Name = $"{item.MachineName}\\{item.Name}"; - - item.CopyMachineInformation(copyFrom); - datFile.Items.Add(parent, item); - } - } - - // All other that would be missing to subfolder of parent - else if (!datFile.Items[parent].Contains(item)) - { - if (subfolder) - item.Name = $"{item.MachineName}\\{item.Name}"; - - item.CopyMachineInformation(copyFrom); - datFile.Items.Add(parent, item); - } - } - - // Then, remove the old game so it's not picked up by the writer - datFile.Items.Remove(game); - } - } - - /// - /// Remove all BIOS and device sets - /// - /// DatFile to filter - private void RemoveBiosAndDeviceSets(DatFile datFile) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - if (datFile.Items[game].Count > 0 - && (datFile.Items[game][0].MachineType.HasFlag(MachineType.Bios) - || datFile.Items[game][0].MachineType.HasFlag(MachineType.Device))) - { - datFile.Items.Remove(game); - } - } - } - - /// - /// Use romof tags to remove bios roms from children - /// - /// DatFile to filter - /// True if only child Bios sets are touched, false for non-bios sets (default) - private void RemoveBiosRomsFromChild(DatFile datFile, bool bios = false) - { - // Loop through the romof tags - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game has no items in it, we want to continue - if (datFile.Items[game].Count == 0) - continue; - - // If the game (is/is not) a bios, we want to continue - if (bios ^ datFile.Items[game][0].MachineType.HasFlag(MachineType.Bios)) - continue; - - // Determine if the game has a parent or not - string parent = null; - if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].RomOf)) - parent = datFile.Items[game][0].RomOf; - - // If the parent doesnt exist, we want to continue - if (string.IsNullOrWhiteSpace(parent)) - continue; - - // If the parent doesn't have any items, we want to continue - if (datFile.Items[parent].Count == 0) - continue; - - // If the parent exists and has items, we remove the items that are in the parent from the current game - List parentItems = datFile.Items[parent]; - foreach (DatItem item in parentItems) - { - DatItem datItem = (DatItem)item.Clone(); - while (datFile.Items[game].Contains(datItem)) - { - datFile.Items.Remove(game, datItem); - } - } - } - } - - /// - /// Use cloneof tags to remove roms from the children - /// - /// DatFile to filter - private void RemoveRomsFromChild(DatFile datFile) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - // If the game has no items in it, we want to continue - if (datFile.Items[game].Count == 0) - continue; - - // Determine if the game has a parent or not - string parent = null; - if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].CloneOf)) - parent = datFile.Items[game][0].CloneOf; - - // If the parent doesnt exist, we want to continue - if (string.IsNullOrWhiteSpace(parent)) - continue; - - // If the parent doesn't have any items, we want to continue - if (datFile.Items[parent].Count == 0) - continue; - - // If the parent exists and has items, we remove the parent items from the current game - List parentItems = datFile.Items[parent]; - foreach (DatItem item in parentItems) - { - DatItem datItem = (DatItem)item.Clone(); - while (datFile.Items[game].Contains(datItem)) - { - datFile.Items.Remove(game, datItem); - } - } - - // Now we want to get the parent romof tag and put it in each of the remaining items - List items = datFile.Items[game]; - string romof = datFile.Items[parent][0].RomOf; - foreach (DatItem item in items) - { - item.RomOf = romof; - } - } - } - - /// - /// Remove all romof and cloneof tags from all games - /// - /// DatFile to filter - private void RemoveTagsFromChild(DatFile datFile) - { - List games = datFile.Items.Keys.OrderBy(g => g).ToList(); - foreach (string game in games) - { - List items = datFile.Items[game]; - foreach (DatItem item in items) - { - item.CloneOf = null; - item.RomOf = null; - item.SampleOf = null; - } - } - } - - #endregion - - #region Manipulation - - /// - /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof - /// - /// DatFile to filter - private void MachineDescriptionToName(DatFile datFile) - { - try - { - // First we want to get a mapping for all games to description - ConcurrentDictionary mapping = new ConcurrentDictionary(); - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - foreach (DatItem item in items) - { - // If the key mapping doesn't exist, add it - mapping.TryAdd(item.MachineName, item.MachineDescription.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); - } - }); - - // Now we loop through every item and update accordingly - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - List newItems = new List(); - foreach (DatItem item in items) - { - // Update machine name - if (!string.IsNullOrWhiteSpace(item.MachineName) && mapping.ContainsKey(item.MachineName)) - item.MachineName = mapping[item.MachineName]; - - // Update cloneof - if (!string.IsNullOrWhiteSpace(item.CloneOf) && mapping.ContainsKey(item.CloneOf)) - item.CloneOf = mapping[item.CloneOf]; - - // Update romof - if (!string.IsNullOrWhiteSpace(item.RomOf) && mapping.ContainsKey(item.RomOf)) - item.RomOf = mapping[item.RomOf]; - - // Update sampleof - if (!string.IsNullOrWhiteSpace(item.SampleOf) && mapping.ContainsKey(item.SampleOf)) - item.SampleOf = mapping[item.SampleOf]; - - // Add the new item to the output list - newItems.Add(item); - } - - // Replace the old list of roms with the new one - datFile.Items.Remove(key); - datFile.Items.AddRange(key, newItems); - }); - } - catch (Exception ex) - { - Globals.Logger.Warning(ex.ToString()); - } - } - - /// - /// Filter a DAT using 1G1R logic given an ordered set of regions - /// - /// DatFile to filter - /// - /// In the most technical sense, the way that the region list is being used does not - /// confine its values to be just regions. Since it's essentially acting like a - /// specialized version of the machine name filter, anything that is usually encapsulated - /// in parenthesis would be matched on, including disc numbers, languages, editions, - /// and anything else commonly used. Please note that, unlike other existing 1G1R - /// solutions, this does not have the ability to contain custom mappings of parent - /// to clone sets based on name, nor does it have the ability to match on the - /// Release DatItem type. - /// - private void OneGamePerRegion(DatFile datFile) - { - // For sake of ease, the first thing we want to do is bucket by game - datFile.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); - - // Then we want to get a mapping of all machines to parents - Dictionary> parents = new Dictionary>(); - foreach (string key in datFile.Items.Keys) - { - DatItem item = datFile.Items[key][0]; - - // Match on CloneOf first - if (!string.IsNullOrEmpty(item.CloneOf)) - { - if (!parents.ContainsKey(item.CloneOf.ToLowerInvariant())) - parents.Add(item.CloneOf.ToLowerInvariant(), new List()); - - parents[item.CloneOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); - } - - // Then by RomOf - else if (!string.IsNullOrEmpty(item.RomOf)) - { - if (!parents.ContainsKey(item.RomOf.ToLowerInvariant())) - parents.Add(item.RomOf.ToLowerInvariant(), new List()); - - parents[item.RomOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); - } - - // Otherwise, treat it as a parent - else - { - if (!parents.ContainsKey(item.MachineName.ToLowerInvariant())) - parents.Add(item.MachineName.ToLowerInvariant(), new List()); - - parents[item.MachineName.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant()); - } - } - - // If we have null region list, make it empty - List regions = datFile.Header.RegionList; - if (regions == null) - regions = new List(); - - // Once we have the full list of mappings, filter out games to keep - foreach (string key in parents.Keys) - { - // Find the first machine that matches the regions in order, if possible - string machine = default; - foreach (string region in regions) - { - machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase)); - if (machine != default) - break; - } - - // If we didn't get a match, use the parent - if (machine == default) - machine = key; - - // Remove the key from the list - parents[key].Remove(machine); - - // Remove the rest of the items from this key - parents[key].ForEach(k => datFile.Items.Remove(k)); - } - - // Finally, strip out the parent tags - RemoveTagsFromChild(datFile); - } - - /// - /// Ensure that all roms are in their own game (or at least try to ensure) - /// - /// DatFile to filter - private void OneRomPerGame(DatFile datFile) - { - // Because this introduces subfolders, we need to set the SuperDAT type - datFile.Header.Type = "SuperDAT"; - - // For each rom, we want to update the game to be "/" - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int i = 0; i < items.Count; i++) - { - string[] splitname = items[i].Name.Split('.'); - items[i].MachineName += $"/{string.Join(".", splitname.Take(splitname.Length > 1 ? splitname.Length - 1 : 1))}"; - items[i].Name = Path.GetFileName(items[i].Name); - } - }); - } - - /// - /// Remove fields as per the header - /// - /// DatFile to filter - private void RemoveFieldsFromItems(DatFile datFile) - { - // Output the logging statement - Globals.Logger.User("Removing filtered fields"); - - // Get the array of fields from the header - List fields = datFile.Header.ExcludeFields; - - // Now process all of the roms - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int j = 0; j < items.Count; j++) - { - DatItem item = items[j]; - - // TODO: Switch statement - foreach (Field field in fields) - { - // Machine Fields - if (field == Field.MachineName) - item.MachineName = null; - if (field == Field.Comment) - item.Comment = null; - if (field == Field.Description) - item.MachineDescription = null; - if (field == Field.Year) - item.Year = null; - if (field == Field.Manufacturer) - item.Manufacturer = null; - if (field == Field.Publisher) - item.Publisher = null; - if (field == Field.Category) - item.Category = null; - if (field == Field.RomOf) - item.RomOf = null; - if (field == Field.CloneOf) - item.CloneOf = null; - if (field == Field.SampleOf) - item.SampleOf = null; - if (field == Field.Supported) - item.Supported = null; - if (field == Field.SourceFile) - item.SourceFile = null; - if (field == Field.Runnable) - item.Runnable = null; - if (field == Field.Board) - item.Board = null; - if (field == Field.RebuildTo) - item.RebuildTo = null; - if (field == Field.Devices) - item.Devices = null; - if (field == Field.SlotOptions) - item.SlotOptions = null; - if (field == Field.Infos) - item.Infos = null; - if (field == Field.MachineType) - item.MachineType = MachineType.NULL; - - // Item Fields - if (field == Field.Name) - item.Name = null; - if (field == Field.PartName) - item.PartName = null; - if (field == Field.PartInterface) - item.PartInterface = null; - if (field == Field.Features) - item.Features = null; - if (field == Field.AreaName) - item.AreaName = null; - if (field == Field.AreaSize) - item.AreaSize = null; - if (field == Field.Default) - { - if (item.ItemType == ItemType.BiosSet) - (item as BiosSet).Default = null; - else if (item.ItemType == ItemType.Release) - (item as Release).Default = null; - } - if (field == Field.BiosDescription) - { - if (item.ItemType == ItemType.BiosSet) - (item as BiosSet).Description = null; - } - if (field == Field.Size) - { - if (item.ItemType == ItemType.Rom) - (item as Rom).Size = 0; - } - if (field == Field.CRC) - { - if (item.ItemType == ItemType.Rom) - (item as Rom).CRC = null; - } - if (field == Field.MD5) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).MD5 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).MD5 = null; - } -#if NET_FRAMEWORK - if (field == Field.RIPEMD160) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).RIPEMD160 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).RIPEMD160 = null; - } -#endif - if (field == Field.SHA1) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).SHA1 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).SHA1 = null; - } - if (field == Field.SHA256) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).SHA256 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).SHA256 = null; - } - if (field == Field.SHA384) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).SHA384 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).SHA384 = null; - } - if (field == Field.SHA512) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).SHA512 = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).SHA512 = null; - } - if (field == Field.Merge) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).MergeTag = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).MergeTag = null; - } - if (field == Field.Region) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).Region = null; - else if (item.ItemType == ItemType.Release) - (item as Release).Region = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).Region = null; - } - if (field == Field.Index) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).Index = null; - } - if (field == Field.Writable) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).Writable = null; - } - if (field == Field.Optional) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).Optional = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).Optional = null; - } - if (field == Field.Status) - { - if (item.ItemType == ItemType.Disk) - (item as Disk).ItemStatus = ItemStatus.NULL; - else if (item.ItemType == ItemType.Rom) - (item as Rom).ItemStatus = ItemStatus.NULL; - } - if (field == Field.Language) - { - if (item.ItemType == ItemType.Release) - (item as Release).Language = null; - } - if (field == Field.Date) - { - if (item.ItemType == ItemType.Release) - (item as Release).Date = null; - else if (item.ItemType == ItemType.Rom) - (item as Rom).Date = null; - } - if (field == Field.Bios) - { - if (item.ItemType == ItemType.Rom) - (item as Rom).Bios = null; - } - if (field == Field.Offset) - { - if (item.ItemType == ItemType.Rom) - (item as Rom).Offset = null; - } - if (field == Field.Inverted) - { - if (item.ItemType == ItemType.Rom) - (item as Rom).Inverted = null; - } - } - - items[j] = item; - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, items); - }); - } - - /// - /// Strip the dates from the beginning of scene-style set names - /// - /// DatFile to filter - private void StripSceneDatesFromItems(DatFile datFile) - { - // Output the logging statement - Globals.Logger.User("Stripping scene-style dates"); - - // Set the regex pattern to use - string pattern = @"([0-9]{2}\.[0-9]{2}\.[0-9]{2}-)(.*?-.*?)"; - - // Now process all of the roms - Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => - { - List items = datFile.Items[key]; - for (int j = 0; j < items.Count; j++) - { - DatItem item = items[j]; - if (Regex.IsMatch(item.MachineName, pattern)) - item.MachineName = Regex.Replace(item.MachineName, pattern, "$2"); - - if (Regex.IsMatch(item.MachineDescription, pattern)) - item.MachineDescription = Regex.Replace(item.MachineDescription, pattern, "$2"); - - items[j] = item; - } - - datFile.Items.Remove(key); - datFile.Items.AddRange(key, items); - }); - } - - #endregion - #endregion // Instance Methods } } diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index f675c535..8c5b6691 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using SabreTools.Library.DatFiles; -using SabreTools.Library.Filtering; using SabreTools.Library.Help; using SabreTools.Library.IO; using SabreTools.Library.Tools; @@ -55,7 +54,7 @@ namespace SabreTools.Features { DatFile datdata = DatFile.Create(); datdata.Parse(datfile, 99, keep: true); - Filter.FilterDatFile(datdata, true); + datdata.ApplyFilter(Filter, true); // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) @@ -78,7 +77,7 @@ namespace SabreTools.Features foreach (ParentablePath datfile in datfilePaths) { datdata.Parse(datfile, 99, keep: true); - Filter.FilterDatFile(datdata, true); + datdata.ApplyFilter(Filter, true); } // If we have overridden the header skipper, set it now