Preliminary 1G1R

SabreTools.Library/DatFiles/DatHeader.cs

@@ -135,7 +135,19 @@ namespace SabreTools.Library.DatFiles
         /// Enable "One Rom, One Region (1G1R)" mode
         /// </summary>
         [JsonIgnore]
-        public bool OneRom { get; set; }
+        public bool OneGamePerRegion { get; set; }
+
+        /// <summary>
+        /// Ordered list of regions for "One Rom, One Region (1G1R)" mode
+        /// </summary>
+        [JsonIgnore]
+        public List<string> RegionList { get; set; }
+
+        /// <summary>
+        /// Ensure each rom is in their own game
+        /// </summary>
+        [JsonIgnore]
+        public bool OneRomPerGame { get; set; }
 
         /// <summary>
         /// Keep machines that don't contain any items
@@ -257,7 +269,9 @@ namespace SabreTools.Library.DatFiles
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
                 ExcludeFields = this.ExcludeFields,
-                OneRom = this.OneRom,
+                OneGamePerRegion = this.OneGamePerRegion,
+                RegionList = this.RegionList,
+                OneRomPerGame = this.OneRomPerGame,
                 KeepEmptyGames = this.KeepEmptyGames,
                 SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
@@ -301,7 +315,9 @@ namespace SabreTools.Library.DatFiles
                 ForcePacking = this.ForcePacking,
                 DatFormat = this.DatFormat,
                 ExcludeFields = this.ExcludeFields,
-                OneRom = this.OneRom,
+                OneGamePerRegion = this.OneGamePerRegion,
+                RegionList = this.RegionList,
+                OneRomPerGame = this.OneRomPerGame,
                 KeepEmptyGames = this.KeepEmptyGames,
                 SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
@@ -318,7 +334,7 @@ namespace SabreTools.Library.DatFiles
             {
                 DatFormat = this.DatFormat,
                 ExcludeFields = this.ExcludeFields,
-                OneRom = this.OneRom,
+                OneRomPerGame = this.OneRomPerGame,
                 KeepEmptyGames = this.KeepEmptyGames,
                 SceneDateStrip = this.SceneDateStrip,
                 DedupeRoms = this.DedupeRoms,
@@ -399,7 +415,7 @@ namespace SabreTools.Library.DatFiles
             if (datHeader.ExcludeFields != null)
                 this.ExcludeFields = datHeader.ExcludeFields;
 
-            this.OneRom = datHeader.OneRom;
+            this.OneRomPerGame = datHeader.OneRomPerGame;
             this.KeepEmptyGames = datHeader.KeepEmptyGames;
             this.SceneDateStrip = datHeader.SceneDateStrip;
             this.DedupeRoms = datHeader.DedupeRoms;

SabreTools.Library/DatFiles/Filter.cs

@@ -811,7 +811,8 @@ namespace SabreTools.Library.DatFiles
                 // We remove any blanks, if we aren't supposed to have any
                 if (!datFile.Header.KeepEmptyGames)
                 {
-                    foreach (string key in datFile.Items.Keys)
+                    List<string> possiblyEmptyKeys = datFile.Items.Keys.ToList();
+                    foreach (string key in possiblyEmptyKeys)
                     {
                         List<DatItem> items = datFile.Items[key];
                         if (items == null)
@@ -825,7 +826,8 @@ namespace SabreTools.Library.DatFiles
                 }
 
                 // Loop over every key in the dictionary
-                foreach (string key in datFile.Items.Keys)
+                List<string> keys = datFile.Items.Keys.ToList();
+                foreach (string key in keys)
                 {
                     // For every item in the current key
                     List<DatItem> items = datFile.Items[key];
@@ -881,7 +883,11 @@ namespace SabreTools.Library.DatFiles
                     StripSceneDatesFromItems(datFile);
 
                 // Run the one rom per game logic, if required
-                if (datFile.Header.OneRom)
+                if (datFile.Header.OneGamePerRegion)
+                    OneGamePerRegion(datFile);
+
+                // Run the one rom per game logic, if required
+                if (datFile.Header.OneRomPerGame)
                     OneRomPerGame(datFile);
 
                 // If we are removing fields, do that now
@@ -897,120 +903,6 @@ namespace SabreTools.Library.DatFiles
             return true;
         }
 
-        /// <summary>
-        /// Filter a DatFile outputting to a new DatFile
-        /// </summary>
-        /// <param name="datFile">DatFile to filter</param>
-        /// <param name="useTags">True if DatFile tags override splitting, false otherwise</param>
-        /// <returns>True if the DatFile was filtered, false on error</returns>
-        public DatFile FilterTo(DatFile datFile, bool useTags)
-        {
-            DatFile outDat = DatFile.Create(datFile.Header);
-
-            try
-            {
-                // Process description to machine name
-                if (this.DescriptionAsName)
-                    MachineDescriptionToName(outDat);
-
-                // If we are using tags from the DAT, set the proper input for split type unless overridden
-                if (useTags && this.InternalSplit == SplitType.None)
-                    this.InternalSplit = outDat.Header.ForceMerging.AsSplitType();
-
-                // Run internal splitting
-                ProcessSplitType(outDat, this.InternalSplit);
-
-                // We remove any blanks, if we aren't supposed to have any
-                if (!outDat.Header.KeepEmptyGames)
-                {
-                    foreach (string key in outDat.Items.Keys)
-                    {
-                        List<DatItem> items = outDat.Items[key];
-                        if (items == null)
-                            continue;
-
-                        List<DatItem> newitems = items.Where(i => i.ItemType != ItemType.Blank).ToList();
-
-                        outDat.Items.Remove(key);
-                        outDat.Items.AddRange(key, newitems);
-                    }
-                }
-
-                // Loop over every key in the dictionary
-                foreach (string key in datFile.Items.Keys)
-                {
-                    // For every item in the current key
-                    List<DatItem> items = datFile.Items[key];
-                    List<DatItem> newitems = new List<DatItem>();
-                    foreach (DatItem item in items)
-                    {
-                        // If the rom passes the filter, include it
-                        if (ItemPasses(item))
-                        {
-                            // If we're stripping unicode characters, do so from all relevant things
-                            if (this.RemoveUnicode)
-                            {
-                                item.Name = Sanitizer.RemoveUnicodeCharacters(item.Name);
-                                item.MachineName = Sanitizer.RemoveUnicodeCharacters(item.MachineName);
-                                item.MachineDescription = Sanitizer.RemoveUnicodeCharacters(item.MachineDescription);
-                            }
-
-                            // If we're in cleaning mode, do so from all relevant things
-                            if (this.Clean)
-                            {
-                                item.MachineName = Sanitizer.CleanGameName(item.MachineName);
-                                item.MachineDescription = Sanitizer.CleanGameName(item.MachineDescription);
-                            }
-
-                            // If we are in single game mode, rename all games
-                            if (this.Single)
-                                item.MachineName = "!";
-
-                            // If we are in NTFS trim mode, trim the game name
-                            if (this.Trim)
-                            {
-                                // Windows max name length is 260
-                                int usableLength = 260 - item.MachineName.Length - this.Root.Length;
-                                if (item.Name.Length > usableLength)
-                                {
-                                    string ext = Path.GetExtension(item.Name);
-                                    item.Name = item.Name.Substring(0, usableLength - ext.Length);
-                                    item.Name += ext;
-                                }
-                            }
-
-                            // Lock the list and add the item back
-                            lock (newitems)
-                            {
-                                newitems.Add(item);
-                            }
-                        }
-                    }
-
-                    outDat.Items.AddRange(key, newitems);
-                }
-
-                // If we are removing scene dates, do that now
-                if (outDat.Header.SceneDateStrip)
-                    StripSceneDatesFromItems(outDat);
-
-                // Run the one rom per game logic, if required
-                if (outDat.Header.OneRom)
-                    OneRomPerGame(outDat);
-
-                // If we are removing fields, do that now
-                if (RemoveFields)
-                    RemoveFieldsFromItems(outDat);
-            }
-            catch (Exception ex)
-            {
-                Globals.Logger.Error(ex.ToString());
-                return null;
-            }
-
-            return outDat;
-        }
-
         /// <summary>
         /// Check to see if a DatItem passes the filter
         /// </summary>
@@ -1124,28 +1016,34 @@ namespace SabreTools.Library.DatFiles
                 return false;
 
             // Filter on devices
-            bool anyPositiveDevice = false;
+            if (item.Devices != null)
-            bool anyNegativeDevice = false;
-            foreach (string device in item.Devices)
             {
-                anyPositiveDevice |= this.Devices.MatchesPositiveSet(device) == true;
+                bool anyPositiveDevice = false;
-                anyNegativeDevice |= this.Devices.MatchesNegativeSet(device) == false;
+                bool anyNegativeDevice = false;
-            }
+                foreach (string device in item.Devices)
+                {
+                    anyPositiveDevice |= this.Devices.MatchesPositiveSet(device) == true;
+                    anyNegativeDevice |= this.Devices.MatchesNegativeSet(device) == false;
+                }
 
-            if (!anyPositiveDevice || anyNegativeDevice)
+                if (!anyPositiveDevice || anyNegativeDevice)
-                return false;
+                    return false;
+            }
 
             // Filter on slot options
-            bool anyPositiveSlotOption = false;
+            if (item.SlotOptions != null)
-            bool anyNegativeSlotOption = false;
-            foreach (string device in item.SlotOptions)
             {
-                anyPositiveSlotOption |= this.SlotOptions.MatchesPositiveSet(device) == true;
+                bool anyPositiveSlotOption = false;
-                anyNegativeSlotOption |= this.SlotOptions.MatchesNegativeSet(device) == false;
+                bool anyNegativeSlotOption = false;
-            }
+                foreach (string device in item.SlotOptions)
+                {
+                    anyPositiveSlotOption |= this.SlotOptions.MatchesPositiveSet(device) == true;
+                    anyNegativeSlotOption |= this.SlotOptions.MatchesNegativeSet(device) == false;
+                }
 
-            if (!anyPositiveSlotOption || anyNegativeSlotOption)
+                if (!anyPositiveSlotOption || anyNegativeSlotOption)
-                return false;
+                    return false;
+            }
 
             // Filter on machine type
             if (this.MachineTypes.MatchesPositive(MachineType.NULL, item.MachineType) == false)
@@ -1774,7 +1672,8 @@ namespace SabreTools.Library.DatFiles
         /// Use cloneof tags to add roms to the parents, removing the child sets in the process
         /// </summary>
         /// <param name="datFile">DatFile to filter</param>
-        private void AddRomsFromChildren(DatFile datFile)
+        /// <param name="subfolder">True to add DatItems to subfolder of parent (not including Disk), false otherwise</param>
+        private void AddRomsFromChildren(DatFile datFile, bool subfolder = true)
         {
             List<string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
             foreach (string game in games)
@@ -1837,7 +1736,9 @@ namespace SabreTools.Library.DatFiles
                         // If the merge tag exists but the parent doesn't contain it, add to subfolder of parent
                         else if (rom.MergeTag != null && !datFile.Items[parent].Select(i => i.Name).Contains(rom.MergeTag))
                         {
-                            item.Name = $"{item.MachineName}\\{item.Name}";
+                            if (subfolder)
+                                item.Name = $"{item.MachineName}\\{item.Name}";
+
                             item.CopyMachineInformation(copyFrom);
                             datFile.Items.Add(parent, item);
                         }
@@ -1845,7 +1746,9 @@ namespace SabreTools.Library.DatFiles
                         // If the parent doesn't already contain this item, add to subfolder of parent
                         else if (!datFile.Items[parent].Contains(item))
                         {
-                            item.Name = $"{item.MachineName}\\{item.Name}";
+                            if (subfolder)
+                                item.Name = $"{item.MachineName}\\{item.Name}";
+
                             item.CopyMachineInformation(copyFrom);
                             datFile.Items.Add(parent, item);
                         }
@@ -1854,7 +1757,9 @@ namespace SabreTools.Library.DatFiles
                     // All other that would be missing to subfolder of parent
                     else if (!datFile.Items[parent].Contains(item))
                     {
-                        item.Name = $"{item.MachineName}\\{item.Name}";
+                        if (subfolder)
+                            item.Name = $"{item.MachineName}\\{item.Name}";
+
                         item.CopyMachineInformation(copyFrom);
                         datFile.Items.Add(parent, item);
                     }
@@ -2056,11 +1961,97 @@ namespace SabreTools.Library.DatFiles
             }
         }
 
+        /// <summary>
+        /// Filter a DAT using 1G1R logic given an ordered set of regions
+        /// </summary>
+        /// <param name="datFile">DatFile to filter</param>
+        private void OneGamePerRegion(DatFile datFile)
+        {
+            // For sake of ease, the first thing we want to do is bucket by game
+            datFile.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
+
+            // Then we want to get a mapping of all games to parents
+            Dictionary<string, List<string>> parents = new Dictionary<string, List<string>>();
+            foreach (string key in datFile.Items.Keys)
+            {
+                DatItem item = datFile.Items[key][0];
+
+                if (!string.IsNullOrEmpty(item.CloneOf))
+                {
+                    if (!parents.ContainsKey(item.CloneOf.ToLowerInvariant()))
+                        parents.Add(item.CloneOf.ToLowerInvariant(), new List<string>());
+
+                    parents[item.CloneOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant());
+                }
+                else if (!string.IsNullOrEmpty(item.RomOf))
+                {
+                    if (!parents.ContainsKey(item.RomOf.ToLowerInvariant()))
+                        parents.Add(item.RomOf.ToLowerInvariant(), new List<string>());
+
+                    parents[item.RomOf.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant());
+                }
+                else
+                {
+                    if (!parents.ContainsKey(item.MachineName.ToLowerInvariant()))
+                        parents.Add(item.MachineName.ToLowerInvariant(), new List<string>());
+
+                    parents[item.MachineName.ToLowerInvariant()].Add(item.MachineName.ToLowerInvariant());
+                }
+            }
+
+            // If we have null region list, make it empty
+            List<string> regions = datFile.Header.RegionList;
+            if (regions == null)
+                regions = new List<string>();
+
+            // Once we have the full list of mappings, get a list of game names to keep
+            List<string> keepMachines = new List<string>();
+            foreach (string key in parents.Keys)
+            {
+                string keepMachine = null;
+
+                // Loop over each region to see if we have a match first
+                foreach (string region in regions)
+                {
+                    // Loop over each machine to see if we have a region match
+                    foreach (string machine in parents[key])
+                    {
+                        if (Regex.IsMatch(machine, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase))
+                        {
+                            keepMachine = machine;
+                            break;
+                        }
+                    }
+
+                    // Break out further if we have something set
+                    if (keepMachine != null)
+                        break;
+                }
+
+                // If we didn't match, use the parent by default
+                if (keepMachine == null)
+                    keepMachine = key;
+
+                keepMachines.Add(keepMachine);
+            }
+
+            // Now that we have a list of machines to keep, weed out the ones that don't match
+            List<string> currentKeys = datFile.Items.Keys.ToList();
+            for (int i = 0; i < currentKeys.Count; i++)
+            {
+                string key = currentKeys[i];
+                if (!keepMachines.Contains(key))
+                    datFile.Items.Remove(key);
+            }
+
+            // Finally, strip out the parent tags
+            RemoveTagsFromChild(datFile);
+        }
+
         /// <summary>
         /// Ensure that all roms are in their own game (or at least try to ensure)
         /// </summary>
         /// <param name="datFile">DatFile to filter</param>
-        /// TODO: This is incorrect for the actual 1G1R logic... this is actually just silly
         private void OneRomPerGame(DatFile datFile)
         {
             // For each rom, we want to update the game to be "<game name>/<rom name>"

SabreTools.Library/README.1ST

@@ -297,6 +297,17 @@ Options:
 		  Exclude any valid item or machine field from outputs. Examples
 		  include: romof, publisher, and offset.
 
+	  -1g1r, --one-game-per-region   Try to ensure one game per region
+		  This allows users to input a list of regions to use to filter on
+		  in order so only one game from each set of parent and clones will be
+		  included. This requires either cloneof or romof tags to function
+		  properly.
+
+  		  -reg, --region		 Add a region for 1G1R
+			  Add a region (in order) for use with 1G1R filtering. If this is
+			  not supplied, then by default, only parent sets will be included
+			  in the output. Multiple instances of this flag are allowed.
+
 	  -orpg, --one-rom-per-game   Try to ensure each rom has its own game
 		  In some cases, it is beneficial to have every rom put into its own
 		  output set as a subfolder of the original parent. This flag enables
@@ -1037,6 +1048,17 @@ Options:
 		  Exclude any valid item or machine field from outputs. Examples
 		  include: romof, publisher, and offset.
 
+	  -1g1r, --one-game-per-region   Try to ensure one game per region
+		  This allows users to input a list of regions to use to filter on
+		  in order so only one game from each set of parent and clones will be
+		  included. This requires either cloneof or romof tags to function
+		  properly.
+
+  		  -reg, --region		 Add a region for 1G1R
+			  Add a region (in order) for use with 1G1R filtering. If this is
+			  not supplied, then by default, only parent sets will be included
+			  in the output. Multiple instances of this flag are allowed.
+
 	  -orpg, --one-rom-per-game   Try to ensure each rom has its own game
 		  In some cases, it is beneficial to have every rom put into its own
 		  output set as a subfolder of the original parent. This flag enables

SabreTools/SabreTools.Help.cs

@@ -596,7 +596,20 @@ namespace SabreTools
             }
         }
 
-        // TODO: Update this later to be actual 1G1R functionality instead
+        public const string OneGamePerRegionValue = "one-game-per-region";
+        private static Feature OneGamePerRegionFlag
+        {
+            get
+            {
+                return new Feature(
+                    OneGamePerRegionValue,
+                    new List<string>() { "-1g1r", "--one-game-per-region" },
+                    "Try to ensure one game per user-defined region",
+                    FeatureType.Flag,
+                    longDescription: "This allows users to input a list of regions to use to filter on in order so only one game from each set of parent and clones will be included. This requires either cloneof or romof tags to function properly.");
+            }
+        }
+
         public const string OneRomPerGameValue = "one-rom-per-game";
         private static Feature OneRomPerGameFlag
         {
@@ -1613,6 +1626,20 @@ Possible values are:
             }
         }
 
+        public const string RegionListValue = "region";
+        private static Feature RegionListInput
+        {
+            get
+            {
+                return new Feature(
+                    RegionListValue,
+                    new List<string>() { "-reg", "--region" },
+                    "Add a region for 1G1R",
+                    FeatureType.List,
+                    longDescription: "Add a region (in order) for use with 1G1R filtering. If this is not supplied, then by default, only parent sets will be included in the output. Multiple instances of this flag are allowed.");
+            }
+        }
+
         public const string ReportTypeListValue = "report-type";
         private static Feature ReportTypeListInput
         {
@@ -2204,10 +2231,12 @@ Some special strings that can be used:
                     Homepage = GetString(features, HomepageStringValue),
                     KeepEmptyGames = GetBoolean(features, KeepEmptyGamesValue),
                     Name = GetString(features, NameStringValue),
-                    OneRom = GetBoolean(features, OneRomPerGameValue),
+                    OneGamePerRegion = GetBoolean(features, OneGamePerRegionValue),
+                    OneRomPerGame = GetBoolean(features, OneRomPerGameValue),
                     Postfix = GetString(features, PostfixStringValue),
                     Prefix = GetString(features, PrefixStringValue),
                     Quotes = GetBoolean(features, QuotesValue),
+                    RegionList = GetList(features, RegionListValue),
                     RemoveExtension = GetBoolean(features, RemoveExtensionsValue),
                     ReplaceExtension = GetString(features, ReplaceExtensionStringValue),
                     Romba = GetBoolean(features, RombaValue),
@@ -2804,6 +2833,8 @@ Some special strings that can be used:
                 AddFeature(CommentStringInput);
                 AddFeature(SuperdatFlag);
                 AddFeature(ExcludeFieldListInput);
+                AddFeature(OneGamePerRegionFlag);
+                this[OneGamePerRegionFlag].AddFeature(RegionListInput);
                 AddFeature(OneRomPerGameFlag);
                 AddFeature(SceneDateStripFlag);
                 AddFeature(AddBlankFilesFlag);
@@ -3303,6 +3334,8 @@ The stats that are outputted are as follows:
                 AddFeature(ForceNodumpStringInput);
                 AddFeature(ForcePackingStringInput);
                 AddFeature(ExcludeFieldListInput);
+                AddFeature(OneGamePerRegionFlag);
+                this[OneGamePerRegionFlag].AddFeature(RegionListInput);
                 AddFeature(OneRomPerGameFlag);
                 AddFeature(KeepEmptyGamesFlag);
                 AddFeature(SceneDateStripFlag);