From 42148b7fc87dcb445d15c2913c9ac62607f21c3c Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 24 Aug 2016 21:19:05 -0700 Subject: [PATCH] [SimpleSort, DatTools, DATFromDir] Add verification This is a rather complex change that allows a very simple fixdat to be created from an input DAT and an output folder. It seems to work as intended so here's hoping that it actually does. It needs more testing, to say the least. --- SabreTools.Helper/Tools/DatTools.cs | 2 +- SabreTools/DATFromDir.cs | 151 +++++++++++++++------ SimpleSort/SimpleSort.cs | 203 +++++++++++++++++++--------- SimpleSort/SimpleSort.csproj | 4 + 4 files changed, 255 insertions(+), 105 deletions(-) diff --git a/SabreTools.Helper/Tools/DatTools.cs b/SabreTools.Helper/Tools/DatTools.cs index 152c630d..2364f14c 100644 --- a/SabreTools.Helper/Tools/DatTools.cs +++ b/SabreTools.Helper/Tools/DatTools.cs @@ -1418,7 +1418,7 @@ namespace SabreTools.Helper foreach (Rom rom in roms) { count++; - string newkey = (norename ? "" : rom.Metadata.SystemID.ToString().PadLeft(10, '0') + "-" + rom.Metadata.SourceID.ToString().PadLeft(10, '0') + "-") + rom.Game.ToLowerInvariant(); + string newkey = (norename ? "" : rom.Metadata.SystemID.ToString().PadLeft(10, '0') + "-" + rom.Metadata.SourceID.ToString().PadLeft(10, '0') + "-") + (String.IsNullOrEmpty(rom.Game) ? "" : rom.Game.ToLowerInvariant()); if (sortable.ContainsKey(newkey)) { sortable[newkey].Add(rom); diff --git a/SabreTools/DATFromDir.cs b/SabreTools/DATFromDir.cs index 07baef43..31fc6ebf 100644 --- a/SabreTools/DATFromDir.cs +++ b/SabreTools/DATFromDir.cs @@ -25,10 +25,17 @@ namespace SabreTools private bool _bare; private bool _archivesAsFiles; private bool _enableGzip; + private bool _nowrite; // Other required variables private Logger _logger; + // Public variables + public Dat DatData + { + get { return _datdata; } + } + /// /// Create a new DATFromDir object /// @@ -38,10 +45,11 @@ namespace SabreTools /// True if SHA-1 hashes should be skipped over, false otherwise /// True if the date should be omitted from the DAT, false otherwise /// True if archives should be treated as files, false otherwise - /// True if GZIP archives should be treated as files, false otherwise> - /// Logger object for console and file output + /// True if GZIP archives should be treated as files, false otherwise /// Name of the directory to create a temp folder in (blank is current directory) - public DATFromDir(List inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger) + /// True if the file should not be written out, false otherwise (default) + /// Logger object for console and file output + public DATFromDir(List inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger, bool nowrite = false) { _inputs = inputs; _datdata = datdata; @@ -52,6 +60,7 @@ namespace SabreTools _enableGzip = enableGzip; _tempDir = tempDir; _logger = logger; + _nowrite = nowrite; } /// @@ -98,10 +107,18 @@ namespace SabreTools _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")"); } - // Create and open the output file for writing - FileStream fs = File.Create(Style.CreateOutfileName(Environment.CurrentDirectory, _datdata)); - StreamWriter sw = new StreamWriter(fs, Encoding.UTF8); - sw.AutoFlush = true; + StreamWriter sw; + if (_nowrite) + { + sw = new StreamWriter(new MemoryStream()); + } + else + { + // Create and open the output file for writing + FileStream fs = File.Create(Style.CreateOutfileName(Environment.CurrentDirectory, _datdata)); + sw = new StreamWriter(fs, Encoding.UTF8); + sw.AutoFlush = true; + } // Write out the initial file header Output.WriteHeader(sw, _datdata, _logger); @@ -227,8 +244,10 @@ namespace SabreTools // Now output any empties to the stream (if not in Romba mode) if (!_datdata.Romba) { - foreach (List roms in _datdata.Roms.Values) + List keys = _datdata.Roms.Keys.ToList(); + foreach (string key in keys) { + List roms = _datdata.Roms[key]; for (int i = 0; i < roms.Count; i++) { Rom rom = roms[i]; @@ -244,23 +263,40 @@ namespace SabreTools rom.SHA1 = Constants.SHA1Zero; } - // If we have a different game and we're not at the start of the list, output the end of last item - int last = 0; - if (lastparent != null && lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + if (_nowrite) { - Output.WriteEndGame(sw, rom, new List(), new List(), lastparent, _datdata, 0, out last, _logger); + string inkey = rom.Size + "-" + rom.CRC; + if (_datdata.Roms.ContainsKey(inkey)) + { + _datdata.Roms[inkey].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + _datdata.Roms.Add(inkey, temp); + } } - - // If we have a new game, output the beginning of the new item - if (lastparent == null || lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + else { - Output.WriteStartGame(sw, rom, new List(), lastparent, _datdata, 0, last, _logger); - } + // If we have a different game and we're not at the start of the list, output the end of last item + int last = 0; + if (lastparent != null && lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + { + Output.WriteEndGame(sw, rom, new List(), new List(), lastparent, _datdata, 0, out last, _logger); + } - // Write out the rom data - if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile) - { - Output.WriteRomData(sw, rom, lastparent, _datdata, 0, _logger); + // If we have a new game, output the beginning of the new item + if (lastparent == null || lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + { + Output.WriteStartGame(sw, rom, new List(), lastparent, _datdata, 0, last, _logger); + } + + // Write out the rom data + if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile) + { + Output.WriteRomData(sw, rom, lastparent, _datdata, 0, _logger); + } } lastparent = rom.Game; @@ -304,9 +340,27 @@ namespace SabreTools if (rom.Name != null) { int last = 0; - Output.WriteStartGame(sw, rom, new List(), "", _datdata, 0, 0, _logger); - Output.WriteRomData(sw, rom, "", _datdata, 0, _logger); - Output.WriteEndGame(sw, rom, new List(), new List(), "", _datdata, 0, out last, _logger); + + if (_nowrite) + { + string key = rom.Size + "-" + rom.CRC; + if (_datdata.Roms.ContainsKey(key)) + { + _datdata.Roms[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + _datdata.Roms.Add(key, temp); + } + } + else + { + Output.WriteStartGame(sw, rom, new List(), "", _datdata, 0, 0, _logger); + Output.WriteRomData(sw, rom, "", _datdata, 0, _logger); + Output.WriteEndGame(sw, rom, new List(), new List(), "", _datdata, 0, out last, _logger); + } } else { @@ -357,10 +411,14 @@ namespace SabreTools _logger.Log(Path.GetFileName(item) + " treated like an archive"); foreach (string entry in Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories)) { + string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar); lastparent = ProcessFile(Path.GetFullPath(entry), sw, Path.GetFullPath(tempdir), - Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) + - Path.GetFileNameWithoutExtension(item) - ), _datdata, lastparent); + (String.IsNullOrEmpty(tempbasepath) + ? "" + : (tempbasepath.Length < _basePath.Length + ? tempbasepath + : tempbasepath.Remove(0, _basePath.Length))) + + Path.GetFileNameWithoutExtension(item), _datdata, lastparent); } // Clear the temp directory @@ -455,21 +513,38 @@ namespace SabreTools rom.Game = rom.Game.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString()); rom.Name = actualitem; - // If we have a different game and we're not at the start of the list, output the end of last item - int last = 0; - if (lastparent != null && lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + if (_nowrite) { - Output.WriteEndGame(sw, rom, new List(), new List(), lastparent, datdata, 0, out last, _logger); + string key = rom.Size + "-" + rom.CRC; + if (_datdata.Roms.ContainsKey(key)) + { + _datdata.Roms[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + _datdata.Roms.Add(key, temp); + } } - - // If we have a new game, output the beginning of the new item - if (lastparent == null || lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + else { - Output.WriteStartGame(sw, rom, new List(), lastparent, datdata, 0, last, _logger); - } + // If we have a different game and we're not at the start of the list, output the end of last item + int last = 0; + if (lastparent != null && lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + { + Output.WriteEndGame(sw, rom, new List(), new List(), lastparent, datdata, 0, out last, _logger); + } - // Write out the rom data - Output.WriteRomData(sw, rom, lastparent, datdata, 0, _logger); + // If we have a new game, output the beginning of the new item + if (lastparent == null || lastparent.ToLowerInvariant() != rom.Game.ToLowerInvariant()) + { + Output.WriteStartGame(sw, rom, new List(), lastparent, datdata, 0, last, _logger); + } + + // Write out the rom data + Output.WriteRomData(sw, rom, lastparent, datdata, 0, _logger); + } _logger.User("File added: " + actualitem + Environment.NewLine); return rom.Game; diff --git a/SimpleSort/SimpleSort.cs b/SimpleSort/SimpleSort.cs index ae16e54f..95143ecb 100644 --- a/SimpleSort/SimpleSort.cs +++ b/SimpleSort/SimpleSort.cs @@ -270,21 +270,19 @@ namespace SabreTools Dat datdata = new Dat(); foreach (string datfile in datfiles) { - datdata = DatTools.Parse(datfile, 0, 0, datdata, logger); + datdata = DatTools.Parse(datfile, 99, 99, datdata, logger); } SimpleSort ss = new SimpleSort(datdata, inputs, outdir, tempdir, quickScan, toFolder, verify, sevenzip, gz, rar, zip, logger); - ss.RebuildToOutput(); + ss.StartProcessing(); } /// - /// Process the DAT and find all matches in input files and folders + /// Pick the appropriate action based on the inputs /// - /// True if rebuilding was a success, false otherwise - public bool RebuildToOutput() + /// True if success, false otherwise + public bool StartProcessing() { - bool success = true; - // First, check that the output directory exists if (!Directory.Exists(_outdir)) { @@ -302,79 +300,152 @@ namespace SabreTools Output.CleanDirectory(_tempdir); } - // If we're in verification mode, only check the output directory if (_verify) { - // Create a list of files from the output directory - List files = new List(); - foreach (string file in Directory.EnumerateFiles(_outdir, "*", SearchOption.AllDirectories)) + return VerifyDirectory(); + } + else + { + return RebuildToOutput(); + } + } + + /// + /// Process the DAT and verify the output directory + /// + /// True if verification was a success, false otherwise + public bool VerifyDirectory() + { + bool success = true; + + // Enumerate all files from the output directory + List files = new List(); + foreach (string file in Directory.EnumerateFiles(_outdir, "*", SearchOption.AllDirectories)) + { + _logger.Log("File found: '" + file + "'"); + files.Add(Path.GetFullPath(file)); + } + + /* + We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder + */ + + // Then, loop through and check each of the inputs + _logger.User("Processing files:\n"); + DATFromDir dfd = new DATFromDir(files, _datdata, false, false, false, false, true, "", _logger, true); + dfd.Start(); + + // Setup the fixdat + _matched = (Dat)_datdata.CloneHeader(); + _matched.Roms = new Dictionary>(); + _matched.FileName = "fixDat_" + _matched.FileName; + _matched.Name = "fixDat_" + _matched.Name; + _matched.Description = "fixDat_" + _matched.Description; + _matched.OutputFormat = OutputFormat.Xml; + + // Now that all files are parsed, get only files found in directory + bool found = false; + foreach (List roms in _datdata.Roms.Values) + { + List newroms = RomTools.Merge(roms, _logger); + foreach (Rom rom in newroms) { - _logger.Log("File found: '" + file + "'"); - files.Add(Path.GetFullPath(file)); + if (rom.Metadata.SourceID == 99) + { + found = true; + string key = rom.Size + "-" + rom.CRC; + if (_matched.Roms.ContainsKey(key)) + { + _matched.Roms[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + _matched.Roms.Add(key, temp); + } + } } } - // Otherwise, run the rebuilder + // Now output the fixdat to the main folder + if (found) + { + Output.WriteDatfile(_matched, "", _logger, stats: true); + } else { - // Create a list of just files from inputs - List files = new List(); - foreach (string input in _inputs) - { - if (File.Exists(input)) - { - _logger.Log("File found: '" + input + "'"); - files.Add(Path.GetFullPath(input)); - } - else if (Directory.Exists(input)) - { - _logger.Log("Directory found: '" + input + "'"); - foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) - { - _logger.Log("File found: '" + file + "'"); - files.Add(Path.GetFullPath(file)); - } - } - else - { - _logger.Error("'" + input + "' is not a file or directory!"); - } - } - - // Then, loop through and check each of the inputs - _logger.User("Processing files:\n"); - _cursorTop = Console.CursorTop; - for (int i = 0; i < files.Count; i++) - { - success &= RebuildToOutputHelper(files[i], i, files.Count); - Output.CleanDirectory(_tempdir); - } - - // Now one final delete of the temp directory - while (Directory.Exists(_tempdir)) - { - try - { - Directory.Delete(_tempdir, true); - } - catch - { - continue; - } - } - - // Now output the stats for the built files - _logger.ClearBeneath(Constants.HeaderHeight); - Console.SetCursorPosition(0, Constants.HeaderHeight + 1); - _logger.User("Stats of the matched ROMs:"); - Stats.OutputStats(_matched, _logger, true); + _logger.User("No fixDat needed"); } return success; } /// - /// Process an individual file against the DAT + /// Process the DAT and find all matches in input files and folders + /// + /// True if rebuilding was a success, false otherwise + public bool RebuildToOutput() + { + bool success = true; + + // Create a list of just files from inputs + List files = new List(); + foreach (string input in _inputs) + { + if (File.Exists(input)) + { + _logger.Log("File found: '" + input + "'"); + files.Add(Path.GetFullPath(input)); + } + else if (Directory.Exists(input)) + { + _logger.Log("Directory found: '" + input + "'"); + foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) + { + _logger.Log("File found: '" + file + "'"); + files.Add(Path.GetFullPath(file)); + } + } + else + { + _logger.Error("'" + input + "' is not a file or directory!"); + } + } + + // Then, loop through and check each of the inputs + _logger.User("Processing files:\n"); + _cursorTop = Console.CursorTop; + for (int i = 0; i < files.Count; i++) + { + success &= RebuildToOutputHelper(files[i], i, files.Count); + Output.CleanDirectory(_tempdir); + } + + // Now one final delete of the temp directory + while (Directory.Exists(_tempdir)) + { + try + { + Directory.Delete(_tempdir, true); + } + catch + { + continue; + } + } + + // Now output the stats for the built files + _logger.ClearBeneath(Constants.HeaderHeight); + Console.SetCursorPosition(0, Constants.HeaderHeight + 1); + _logger.User("Stats of the matched ROMs:"); + Stats.OutputStats(_matched, _logger, true); + + return success; + } + + /// + /// Process an individual file against the DAT for rebuilding /// /// Name of the input file /// Index of the current file diff --git a/SimpleSort/SimpleSort.csproj b/SimpleSort/SimpleSort.csproj index ba55fdda..ea0aaf7c 100644 --- a/SimpleSort/SimpleSort.csproj +++ b/SimpleSort/SimpleSort.csproj @@ -79,6 +79,10 @@ {225a1afd-0890-44e8-b779-7502665c23a5} SabreTools.Helper + + {3b615702-1866-4d7b-8af1-7b43fd0cc1d0} + SabreTools +