From ebb6529440c48d0cd350c206dd8d907401b3d3c1 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 21 Aug 2020 10:38:42 -0700 Subject: [PATCH] Wire up new flag --- RombaSharp/Features/Diffdat.cs | 4 +- RombaSharp/Features/EDiffdat.cs | 4 +- SabreTools.Library/DatFiles/DatFile.cs | 130 +++++++++++++++++++++-- SabreTools.Library/DatFiles/RomCenter.cs | 2 +- SabreTools.Library/Filtering/ExtraIni.cs | 22 ++++ SabreTools.Library/README.1ST | 15 +++ SabreTools/Features/BaseFeature.cs | 30 ++++++ SabreTools/Features/DatFromDir.cs | 2 + SabreTools/Features/Update.cs | 8 +- SabreTools/Features/Verify.cs | 7 +- 10 files changed, 204 insertions(+), 20 deletions(-) diff --git a/RombaSharp/Features/Diffdat.cs b/RombaSharp/Features/Diffdat.cs index 0cb7432e..6cf9ad31 100644 --- a/RombaSharp/Features/Diffdat.cs +++ b/RombaSharp/Features/Diffdat.cs @@ -67,8 +67,8 @@ in -old DAT file. Ignores those entries in -old that are not in -new."; List basedats = new List { olddat }; // Now run the diff on the inputs - datfile.PopulateUserData(basedats, new Filter()); - datfile.DiffAgainst(dats, outdat, false, new Filter(), false); + datfile.PopulateUserData(basedats, new ExtraIni(), new Filter()); + datfile.DiffAgainst(dats, outdat, false, new ExtraIni(), new Filter(), false); } } } diff --git a/RombaSharp/Features/EDiffdat.cs b/RombaSharp/Features/EDiffdat.cs index 4fec9fbe..5116fe06 100644 --- a/RombaSharp/Features/EDiffdat.cs +++ b/RombaSharp/Features/EDiffdat.cs @@ -60,8 +60,8 @@ namespace RombaSharp.Features List basedats = new List { olddat }; // Now run the diff on the inputs - datfile.PopulateUserData(basedats, new Filter()); - datfile.DiffAgainst(dats, outdat, false, new Filter(), false); + datfile.PopulateUserData(basedats, new ExtraIni(), new Filter()); + datfile.DiffAgainst(dats, outdat, false, new ExtraIni(), new Filter(), false); } } } diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index f5a31d4d..06f42021 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -200,6 +200,7 @@ namespace SabreTools.Library.DatFiles /// Names of the input files /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// List of Fields representing what should be updated [only for base replacement] /// True if descriptions should only be replaced if the game name is the same, false otherwise @@ -207,12 +208,13 @@ namespace SabreTools.Library.DatFiles List inputs, string outDir, bool inplace, + ExtraIni extras, Filter filter, List updateFields, bool onlySame) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - BaseReplace(paths, outDir, inplace, filter, updateFields, onlySame); + BaseReplace(paths, outDir, inplace, extras, filter, updateFields, onlySame); } /// @@ -221,6 +223,7 @@ namespace SabreTools.Library.DatFiles /// Names of the input files /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// List of Fields representing what should be updated [only for base replacement] /// True if descriptions should only be replaced if the game name is the same, false otherwise @@ -228,6 +231,7 @@ namespace SabreTools.Library.DatFiles List inputs, string outDir, bool inplace, + ExtraIni extras, Filter filter, List updateFields, bool onlySame) @@ -240,6 +244,7 @@ namespace SabreTools.Library.DatFiles // First we parse in the DAT internally DatFile intDat = Create(Header.CloneFiltering()); intDat.Parse(path, 1, keep: true); + intDat.ApplyExtras(extras); intDat.ApplyFilter(filter, false /* useTags */); // If we are matching based on DatItem fields of any sort @@ -316,12 +321,19 @@ namespace SabreTools.Library.DatFiles /// Names of the input files /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// True to diff using games, false to use hashes - public void DiffAgainst(List inputs, string outDir, bool inplace, Filter filter, bool useGames) + public void DiffAgainst( + List inputs, + string outDir, + bool inplace, + ExtraIni extras, + Filter filter, + bool useGames) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - DiffAgainst(paths, outDir, inplace, filter, useGames); + DiffAgainst(paths, outDir, inplace, extras, filter, useGames); } /// @@ -330,9 +342,16 @@ namespace SabreTools.Library.DatFiles /// Names of the input files /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// True to diff using games, false to use hashes - public void DiffAgainst(List inputs, string outDir, bool inplace, Filter filter, bool useGames) + public void DiffAgainst( + List inputs, + string outDir, + bool inplace, + ExtraIni extras, + Filter filter, + bool useGames) { // For comparison's sake, we want to use a base ordering if (useGames) @@ -348,6 +367,7 @@ namespace SabreTools.Library.DatFiles // First we parse in the DAT internally DatFile intDat = Create(Header.CloneFiltering()); intDat.Parse(path, 1, keep: true); + intDat.ApplyExtras(extras); intDat.ApplyFilter(filter, false /* useTags */); // For comparison's sake, we want to a the base bucketing @@ -803,21 +823,23 @@ namespace SabreTools.Library.DatFiles /// Populate the user DatData object from the input files /// /// Paths to DATs to parse + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// List of DatData objects representing headers - public List PopulateUserData(List inputs, Filter filter) + public List PopulateUserData(List inputs, ExtraIni extras, Filter filter) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - return PopulateUserData(paths, filter); + return PopulateUserData(paths, extras, filter); } /// /// Populate the user DatData object from the input files /// /// Paths to DATs to parse + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// List of DatData objects representing headers - public List PopulateUserData(List inputs, Filter filter) + public List PopulateUserData(List inputs, ExtraIni extras, Filter filter) { DatFile[] datFiles = new DatFile[inputs.Count]; InternalStopwatch watch = new InternalStopwatch("Processing individual DATs"); @@ -839,7 +861,8 @@ namespace SabreTools.Library.DatFiles AddFromExisting(datFiles[i], true); } - // Now that we have a merged DAT, filter it + // Now that we have a merged DAT, apply extras and filter it + ApplyExtras(extras); ApplyFilter(filter, false /* useTags */); watch.Stop(); @@ -853,11 +876,17 @@ namespace SabreTools.Library.DatFiles /// Names of the input files and/or folders /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level - public void Update(List inputs, string outDir, bool inplace, Filter filter) + public void Update( + List inputs, + string outDir, + bool inplace, + ExtraIni extras, + Filter filter) { List paths = inputs.Select(i => new ParentablePath(i)).ToList(); - Update(paths, outDir, inplace, filter); + Update(paths, outDir, inplace, extras, filter); } /// @@ -866,8 +895,14 @@ namespace SabreTools.Library.DatFiles /// Names of the input files and/or folders /// Optional param for output directory /// True if the output files should overwrite their inputs, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level - public void Update(List inputs, string outDir, bool inplace, Filter filter) + public void Update( + List inputs, + string outDir, + bool inplace, + ExtraIni extras, + Filter filter) { // Iterate over the files foreach (ParentablePath file in inputs) @@ -878,6 +913,7 @@ namespace SabreTools.Library.DatFiles keepext: innerDatdata.Header.DatFormat.HasFlag(DatFormat.TSV) || innerDatdata.Header.DatFormat.HasFlag(DatFormat.CSV) || innerDatdata.Header.DatFormat.HasFlag(DatFormat.SSV)); + innerDatdata.ApplyExtras(extras); innerDatdata.ApplyFilter(filter, false /* useTags */); // Get the correct output path @@ -892,6 +928,69 @@ namespace SabreTools.Library.DatFiles #region Filtering + /// + /// Apply a set of Extra INIs on the DatFile + /// + /// ExtrasIni to use + /// True if the extras were applied, false on error + public bool ApplyExtras(ExtraIni extras) + { + try + { + // Bucket by game first + Items.BucketBy(BucketedBy.Game, DedupeType.None); + + // Create a new set of mappings based on the items + var map = new Dictionary>(); + + // Loop through each of the extras + foreach (ExtraIniItem item in extras.Items) + { + foreach (var mapping in item.Mappings) + { + string key = mapping.Key; + List machineNames = mapping.Value; + + // If we have the root folder, assume boolean + if (string.Equals(key, "ROOT_FOLDER")) + key = "true"; + + // Loop through the machines and add the new mappings + foreach (string machine in machineNames) + { + if (!map.ContainsKey(machine)) + map[machine] = new Dictionary(); + + map[machine][item.Field] = key; + } + } + } + + // Now apply the new set of mappings + foreach (string key in map.Keys) + { + // If the key doesn't exist, continue + if (!Items.ContainsKey(key)) + continue; + + List datItems = Items[key]; + var mappings = map[key]; + + foreach (var datItem in datItems) + { + datItem.SetFields(mappings); + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + /// /// Apply a Filter on the DatFile /// @@ -2005,6 +2104,7 @@ namespace SabreTools.Library.DatFiles /// True if dates should be archived for all files, false otherwise /// Output directory to /// True if files should be copied to the temp directory before hashing, false otherwise + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// True if DatFile tags override splitting, false otherwise /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually @@ -2017,6 +2117,7 @@ namespace SabreTools.Library.DatFiles bool addBlanks = false, bool addDate = false, bool copyFiles = false, + ExtraIni extras = null, Filter filter = null, bool useTags = false) { @@ -2102,6 +2203,10 @@ namespace SabreTools.Library.DatFiles if (Globals.TempDir != Path.GetTempPath()) DirectoryExtensions.TryDelete(Globals.TempDir); + // If we have valid extras, perform the application now + if (extras != null && extras != default(ExtraIni)) + ApplyExtras(extras); + // If we have a valid filter, perform the filtering now if (filter != null && filter != default(Filter)) ApplyFilter(filter, useTags); @@ -3092,6 +3197,7 @@ namespace SabreTools.Library.DatFiles /// True if only hashes should be checked, false for full file information /// True to enable external scanning of archives, false otherwise /// TreatAsFiles representing CHD and Archive scanning + /// ExtraIni object to apply to the DatFile /// Filter object to be passed to the DatItem level /// True if verification was a success, false otherwise public bool VerifyGeneric( @@ -3100,6 +3206,7 @@ namespace SabreTools.Library.DatFiles bool hashOnly = false, bool quickScan = false, TreatAsFiles asFiles = 0x00, + ExtraIni extras = null, Filter filter = null) { // TODO: We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder @@ -3115,6 +3222,7 @@ namespace SabreTools.Library.DatFiles quickScan ? Hash.SecureHashes : Hash.DeepHashes, bare: true, asFiles: asFiles, + extras: extras, filter: filter); } diff --git a/SabreTools.Library/DatFiles/RomCenter.cs b/SabreTools.Library/DatFiles/RomCenter.cs index 31918b85..5f6c94bd 100644 --- a/SabreTools.Library/DatFiles/RomCenter.cs +++ b/SabreTools.Library/DatFiles/RomCenter.cs @@ -44,7 +44,7 @@ namespace SabreTools.Library.DatFiles if (ir == null) return; - // Otherwise, read teh file to the end + // Otherwise, read the file to the end try { ir.ReadNextLine(); diff --git a/SabreTools.Library/Filtering/ExtraIni.cs b/SabreTools.Library/Filtering/ExtraIni.cs index 6c91e3ae..b5f81a05 100644 --- a/SabreTools.Library/Filtering/ExtraIni.cs +++ b/SabreTools.Library/Filtering/ExtraIni.cs @@ -4,9 +4,31 @@ namespace SabreTools.Library.Filtering { public class ExtraIni { + #region Fields + /// /// List of extras to apply /// public List Items { get; set; } = new List(); + + #endregion + + #region Extras Population + + /// + /// Populate item using field:file inputs + /// + /// Field and file combinations + public void PopulateFromList(List inputs) + { + foreach (string input in inputs) + { + ExtraIniItem item = new ExtraIniItem(); + item.PopulateFromInput(input); + Items.Add(item); + } + } + + #endregion } } diff --git a/SabreTools.Library/README.1ST b/SabreTools.Library/README.1ST index c8724bca..26193faa 100644 --- a/SabreTools.Library/README.1ST +++ b/SabreTools.Library/README.1ST @@ -360,6 +360,11 @@ Options: compare against the input DATs. This flag forces all CHDs to be treated like regular files. + -ini=, --extra-ini= Apply a MAME INI for given field(s) + Apply any valid MAME INI for any valid field in the DatFile. Inputs are + of the form 'Field:path\to\ini'. Multiple instances of this flag are + allowed. + -fi=, --filter= Filter a game/rom field with the given value(s) Filter any valid item or machine field from inputs. Filters are input in the form 'key:value' or '!key:value', where the '!' signifies 'not' @@ -1312,6 +1317,11 @@ Options: second time, this will skip writing it. This can often speed up the output process. [Both diff-cascade and diff-reverse-cascade] + -ini=, --extra-ini= Apply a MAME INI for given field(s) + Apply any valid MAME INI for any valid field in the DatFile. Inputs are + of the form 'Field:path\to\ini'. Multiple instances of this flag are + allowed. + -fi=, --filter= Filter a game/rom field with the given value(s) Filter any valid item or machine field from inputs. Filters are input in the form 'key:value' or '!key:value', where the '!' signifies 'not' @@ -1597,6 +1607,11 @@ Options: parent sets based on the cloneof and romof tags as well as device references. This is incompatible with the other --dat-X flags. + -ini=, --extra-ini= Apply a MAME INI for given field(s) + Apply any valid MAME INI for any valid field in the DatFile. Inputs are + of the form 'Field:path\to\ini'. Multiple instances of this flag are + allowed. + -fi=, --filter= Filter a game/rom field with the given value(s) Filter any valid item or machine field from inputs. Filters are input in the form 'key:value' or '!key:value', where the '!' signifies 'not' diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index bee4df30..ced839ed 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -1382,6 +1382,20 @@ namespace SabreTools.Features } } + internal const string ExtraIniListValue = "extra-ini"; + internal static Feature ExtraIniListInput + { + get + { + return new Feature( + ExtraIniListValue, + new List() { "-ini", "--extra-ini" }, + "Apply a MAME INI for given field(s)", + FeatureType.List, + longDescription: "Apply any valid MAME INI for any valid field in the DatFile. Inputs are of the form 'Field:path\\to\\ini'. Multiple instances of this flag are allowed."); + } + } + internal const string FilterListValue = "filter"; internal static Feature FilterListInput { @@ -2274,6 +2288,11 @@ Some special strings that can be used: #region Fields + /// + /// Preconfigured ExtraIni set + /// + protected ExtraIni Extras { get; set; } + /// /// Pre-configured Filter /// @@ -2386,6 +2405,7 @@ Some special strings that can be used: public override void ProcessFeatures(Dictionary features) { // Generic feature flags + Extras = GetExtras(features); Filter = GetFilter(features); Header = GetDatHeader(features); OutputDir = GetString(features, OutputDirStringValue); @@ -2709,6 +2729,16 @@ Some special strings that can be used: return DedupeType.None; } + /// + /// Get ExtraIni from feature list + /// + private ExtraIni GetExtras(Dictionary features) + { + ExtraIni extraIni = new ExtraIni(); + extraIni.PopulateFromList(GetList(features, ExtraIniListValue)); + return extraIni; + } + /// /// Get Filter from feature list /// diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index c4093439..a3418f07 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -44,6 +44,7 @@ namespace SabreTools.Features AddFeature(CopyFilesFlag); AddFeature(HeaderStringInput); AddFeature(ChdsAsFilesFlag); + AddFeature(ExtraIniListInput); AddFilteringFeatures(); AddFeature(TempStringInput); AddFeature(OutputDirStringInput); @@ -85,6 +86,7 @@ namespace SabreTools.Features addBlankFiles, addFileDates, copyFiles, + Extras, Filter); if (success) diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index cfc1ecb3..2585421d 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -87,6 +87,7 @@ namespace SabreTools.Features this[DiffCascadeFlag].AddFeature(SkipFirstOutputFlag); AddFeature(DiffReverseCascadeFlag); this[DiffReverseCascadeFlag].AddFeature(SkipFirstOutputFlag); + AddFeature(ExtraIniListInput); AddFilteringFeatures(); AddFeature(OutputDirStringInput); AddFeature(InplaceFlag); @@ -156,6 +157,7 @@ namespace SabreTools.Features inputFileNames, OutputDir, GetBoolean(features, InplaceValue), + Extras, Filter); return; } @@ -178,9 +180,9 @@ namespace SabreTools.Features // Populate using the correct set List datHeaders; if (updateMode.HasFlag(UpdateMode.DiffAgainst) || updateMode.HasFlag(UpdateMode.BaseReplace)) - datHeaders = userInputDat.PopulateUserData(baseFileNames, Filter); + datHeaders = userInputDat.PopulateUserData(baseFileNames, Extras, Filter); else - datHeaders = userInputDat.PopulateUserData(inputFileNames, Filter); + datHeaders = userInputDat.PopulateUserData(inputFileNames, Extras, Filter); // Output only DatItems that are duplicated across inputs if (updateMode.HasFlag(UpdateMode.DiffDupesOnly)) @@ -212,6 +214,7 @@ namespace SabreTools.Features inputFileNames, OutputDir, GetBoolean(features, InplaceValue), + Extras, Filter, GetBoolean(Features, ByGameValue)); } @@ -223,6 +226,7 @@ namespace SabreTools.Features inputFileNames, OutputDir, GetBoolean(features, InplaceValue), + Extras, Filter, updateFields, GetBoolean(features, OnlySameValue)); diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 21882a3a..152b754d 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -31,6 +31,7 @@ namespace SabreTools.Features AddFeature(ChdsAsFilesFlag); AddFeature(IndividualFlag); AddInternalSplitFeatures(); + AddFeature(ExtraIniListInput); AddFilteringFeatures(); } @@ -54,6 +55,7 @@ namespace SabreTools.Features { DatFile datdata = DatFile.Create(); datdata.Parse(datfile, 99, keep: true); + datdata.ApplyExtras(Extras); datdata.ApplyFilter(Filter, true); // Set depot information @@ -67,7 +69,7 @@ namespace SabreTools.Features if (Header.InputDepot.IsActive) datdata.VerifyDepot(Inputs, OutputDir); else - datdata.VerifyGeneric(Inputs, OutputDir, hashOnly, quickScan, asFiles, Filter); + datdata.VerifyGeneric(Inputs, OutputDir, hashOnly, quickScan, asFiles, Extras, Filter); } } // Otherwise, process all DATs into the same output @@ -80,6 +82,7 @@ namespace SabreTools.Features foreach (ParentablePath datfile in datfilePaths) { datdata.Parse(datfile, 99, keep: true); + datdata.ApplyExtras(Extras); datdata.ApplyFilter(Filter, true); } @@ -96,7 +99,7 @@ namespace SabreTools.Features if (Header.InputDepot.IsActive) datdata.VerifyDepot(Inputs, OutputDir); else - datdata.VerifyGeneric(Inputs, OutputDir, hashOnly, quickScan, asFiles, Filter); + datdata.VerifyGeneric(Inputs, OutputDir, hashOnly, quickScan, asFiles, Extras, Filter); } } }