diff --git a/SabreTools.Helper/Objects/Dat/DatFile.cs b/SabreTools.Helper/Objects/Dat/DatFile.cs
index f1a3c445..1b32cc8b 100644
--- a/SabreTools.Helper/Objects/Dat/DatFile.cs
+++ b/SabreTools.Helper/Objects/Dat/DatFile.cs
@@ -384,6 +384,8 @@ namespace SabreTools.Helper
#endregion
+ #region Instance Methods
+
#region Cloning Methods
public object Clone()
@@ -471,55 +473,6 @@ namespace SabreTools.Helper
#region DAT Parsing
- ///
- /// Get what type of DAT the input file is
- ///
- /// Name of the file to be parsed
- /// The OutputFormat corresponding to the DAT
- /// There is currently no differentiation between XML and SabreDAT here
- public static OutputFormat GetOutputFormat(string filename, Logger logger)
- {
- // Limit the output formats based on extension
- string ext = Path.GetExtension(filename).ToLowerInvariant();
- if (ext != ".dat" && ext != ".xml")
- {
- return 0;
- }
-
- // Read the input file, if possible
- logger.Log("Attempting to read file: \"" + filename + "\"");
-
- // Check if file exists
- if (!File.Exists(filename))
- {
- logger.Warning("File '" + filename + "' could not read from!");
- return 0;
- }
-
- try
- {
- StreamReader sr = File.OpenText(filename);
- string first = sr.ReadLine();
- sr.Dispose();
- if (first.Contains("<") && first.Contains(">"))
- {
- return OutputFormat.Xml;
- }
- else if (first.Contains("[") && first.Contains("]"))
- {
- return OutputFormat.RomCenter;
- }
- else
- {
- return OutputFormat.ClrMamePro;
- }
- }
- catch (Exception)
- {
- return 0;
- }
- }
-
///
/// Parse a DAT and return all found games and roms within
///
@@ -602,7 +555,7 @@ namespace SabreTools.Helper
FileName = (String.IsNullOrEmpty(FileName) ? (keepext ? Path.GetFileName(filename) : Path.GetFileNameWithoutExtension(filename)) : FileName);
// If the output type isn't set already, get the internal output type
- OutputFormat = (OutputFormat == 0 ? GetOutputFormat(filename, logger) : OutputFormat);
+ OutputFormat = (OutputFormat == 0 ? FileTools.GetOutputFormat(filename, logger) : OutputFormat);
// Make sure there's a dictionary to read to
if (Files == null)
@@ -611,7 +564,7 @@ namespace SabreTools.Helper
}
// Now parse the correct type of DAT
- switch (GetOutputFormat(filename, logger))
+ switch (FileTools.GetOutputFormat(filename, logger))
{
case OutputFormat.ClrMamePro:
ParseCMP(filename, sysid, srcid, gamename, romname, romtype, sgt, slt, seq, crc, md5, sha1, itemStatus, trim, single, root, logger, keep, clean);
@@ -2338,1227 +2291,6 @@ namespace SabreTools.Helper
#endregion
- #region Populate DAT from Directory
-
- ///
- /// Create a new Dat from a directory
- ///
- /// Base folder to be used in creating the DAT
- /// True if MD5 hashes should be skipped over, false otherwise
- /// True if SHA-1 hashes should be skipped over, false otherwise
- /// True if the date should be omitted from the DAT, false otherwise
- /// True if archives should be treated as files, false otherwise
- /// True if GZIP archives should be treated as files, false otherwise
- /// True if blank items should be created for empty folders, false otherwise
- /// True if dates should be archived for all files, false otherwise
- /// Name of the directory to create a temp folder in (blank is current directory)
- /// True if files should be copied to the temp directory before hashing, false otherwise
- /// Integer representing the maximum amount of parallelization to be used
- /// Logger object for console and file output
- public bool PopulateDatFromDir(string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
- bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
- {
- // If the description is defined but not the name, set the name from the description
- if (String.IsNullOrEmpty(Name) && !String.IsNullOrEmpty(Description))
- {
- Name = Description;
- }
-
- // If the name is defined but not the description, set the description from the name
- else if (!String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
- {
- Description = Name + (bare ? "" : " (" + Date + ")");
- }
-
- // If neither the name or description are defined, set them from the automatic values
- else if (String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
- {
- Name = basePath.Split(Path.DirectorySeparatorChar).Last();
- Description = Name + (bare ? "" : " (" + Date + ")");
- }
-
- // Process the input folder
- logger.Log("Folder found: " + basePath);
-
- // Process the files in all subfolders
- List files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList();
- Parallel.ForEach(files,
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- item =>
- {
- DFDProcessPossibleArchive(item, basePath, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, addBlanks, addDate,
- tempDir, copyFiles, maxDegreeOfParallelism, logger);
- });
-
- // Now find all folders that are empty, if we are supposed to
- if (!Romba && addBlanks)
- {
- List empties = Directory.EnumerateDirectories(basePath, "*", SearchOption.AllDirectories).ToList();
- Parallel.ForEach(empties,
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- dir =>
- {
- if (Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly).Count() == 0)
- {
- // Get the full path for the directory
- string fulldir = Path.GetFullPath(dir);
-
- // Set the temporary variables
- string gamename = "";
- string romname = "";
-
- // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
- if (Type == "SuperDAT")
- {
- gamename = fulldir.Remove(0, basePath.Length + 1);
- romname = "-";
- }
-
- // Otherwise, we want just the top level folder as the game, and the file as everything else
- else
- {
- gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0];
- romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "-");
- }
-
- // Sanitize the names
- if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
- {
- gamename = gamename.Substring(1);
- }
- if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
- {
- gamename = gamename.Substring(0, gamename.Length - 1);
- }
- if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
- {
- romname = romname.Substring(1);
- }
- if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
- {
- romname = romname.Substring(0, romname.Length - 1);
- }
-
- logger.Log("Adding blank empty folder: " + gamename);
- Files["null"].Add(new Rom(romname, gamename));
- }
- });
- }
-
- // Now that we're done, delete the temp folder (if it's not the default)
- logger.User("Cleaning temp folder");
- try
- {
- if (tempDir != Path.GetTempPath())
- {
- Directory.Delete(tempDir, true);
- }
- }
- catch
- {
- // Just absorb the error for now
- }
-
- return true;
- }
-
- ///
- /// Check a given file for hashes, based on current settings
- ///
- /// Filename of the item to be checked
- /// Base folder to be used in creating the DAT
- /// True if MD5 hashes should be skipped over, false otherwise
- /// True if SHA-1 hashes should be skipped over, false otherwise
- /// True if the date should be omitted from the DAT, false otherwise
- /// True if archives should be treated as files, false otherwise
- /// True if GZIP archives should be treated as files, false otherwise
- /// True if blank items should be created for empty folders, false otherwise
- /// True if dates should be archived for all files, false otherwise
- /// Name of the directory to create a temp folder in (blank is current directory)
- /// True if files should be copied to the temp directory before hashing, false otherwise
- /// Integer representing the maximum amount of parallelization to be used
- /// Logger object for console and file output
- private void DFDProcessPossibleArchive(string item, string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
- bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
- {
- // Define the temporary directory
- string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar;
-
- // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
- if (Romba)
- {
- Rom rom = FileTools.GetTorrentGZFileInfo(item, logger);
-
- // If the rom is valid, write it out
- if (rom.Name != null)
- {
- // Add the list if it doesn't exist already
- string key = rom.Size + "-" + rom.CRC;
-
- lock (Files)
- {
- if (!Files.ContainsKey(key))
- {
- Files.Add(key, new List());
- }
-
- Files[key].Add(rom);
- logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
- }
-
- }
- else
- {
- logger.User("File not added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
- return;
- }
-
- return;
- }
-
- // If we're copying files, copy it first and get the new filename
- string newItem = item;
- string newBasePath = basePath;
- if (copyFiles)
- {
- newBasePath = Path.Combine(tempDir, Path.GetRandomFileName());
- newItem = Path.GetFullPath(Path.Combine(newBasePath, Path.GetFullPath(item).Remove(0, basePath.Length + 1)));
- Directory.CreateDirectory(Path.GetDirectoryName(newItem));
- File.Copy(item, newItem, true);
- }
-
- // If both deep hash skip flags are set, do a quickscan
- if (noMD5 && noSHA1)
- {
- ArchiveType? type = FileTools.GetCurrentArchiveType(newItem, logger);
-
- // If we have an archive, scan it
- if (type != null && !archivesAsFiles)
- {
- List extracted = FileTools.GetArchiveFileInfo(newItem, logger);
-
- foreach (Rom rom in extracted)
- {
- DFDProcessFileHelper(newItem,
- rom,
- basePath,
- (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item),
- logger);
- }
- }
- // Otherwise, just get the info on the file itself
- else if (File.Exists(newItem))
- {
- DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
- }
- }
- // Otherwise, attempt to extract the files to the temporary directory
- else
- {
- bool encounteredErrors = FileTools.ExtractArchive(newItem,
- tempSubDir,
- (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
- (!archivesAsFiles && enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External),
- (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
- (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
- logger);
-
- // If the file was an archive and was extracted successfully, check it
- if (!encounteredErrors)
- {
- logger.Log(Path.GetFileName(item) + " treated like an archive");
- List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList();
- Parallel.ForEach(extracted,
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- entry =>
- {
- DFDProcessFile(entry,
- Path.Combine((Type == "SuperDAT"
- ? (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length)
- : ""),
- Path.GetFileNameWithoutExtension(item)),
- tempSubDir,
- noMD5,
- noSHA1,
- addDate,
- logger);
- });
- }
- // Otherwise, just get the info on the file itself
- else if (File.Exists(newItem))
- {
- DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
- }
- }
-
- // Cue to delete the file if it's a copy
- if (copyFiles && item != newItem)
- {
- try
- {
- Directory.Delete(newBasePath, true);
- }
- catch { }
- }
-
- // Delete the sub temp directory
- if (Directory.Exists(tempSubDir))
- {
- Directory.Delete(tempSubDir, true);
- }
- }
-
- ///
- /// Process a single file as a file
- ///
- /// File to be added
- /// Path the represents the parent directory
- /// Parent game to be used
- private void DFDProcessFile(string item, string parent, string basePath, bool noMD5, bool noSHA1, bool addDate, Logger logger)
- {
- logger.Log(Path.GetFileName(item) + " treated like a file");
- Rom rom = FileTools.GetSingleFileInfo(item, noMD5: noMD5, noSHA1: noSHA1, date: addDate);
-
- DFDProcessFileHelper(item, rom, basePath, parent, logger);
- }
-
- ///
- /// Process a single file as a file (with found Rom data)
- ///
- /// File to be added
- /// Rom data to be used to write to file
- /// Path the represents the parent directory
- /// Parent game to be used
- private void DFDProcessFileHelper(string item, DatItem datItem, string basepath, string parent, Logger logger)
- {
- // If the datItem isn't a Rom or Disk, return
- if (datItem.Type != ItemType.Rom && datItem.Type != ItemType.Disk)
- {
- return;
- }
-
- string key = "";
- if (datItem.Type == ItemType.Rom)
- {
- key = ((Rom)datItem).Size + "-" + ((Rom)datItem).CRC;
- }
- else
- {
- key = ((Disk)datItem).MD5;
- }
-
- // Add the list if it doesn't exist already
- lock (Files)
- {
- if (!Files.ContainsKey(key))
- {
- Files.Add(key, new List());
- }
- }
-
- try
- {
- // If the basepath ends with a directory separator, remove it
- if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
- {
- basepath += Path.DirectorySeparatorChar.ToString();
- }
-
- // Make sure we have the full item path
- item = Path.GetFullPath(item);
-
- // Get the data to be added as game and item names
- string gamename = "";
- string romname = "";
-
- // If the parent is blank, then we have a non-archive file
- if (parent == "")
- {
- // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
- if (Type == "SuperDAT")
- {
- gamename = Path.GetDirectoryName(item.Remove(0, basepath.Length));
- romname = Path.GetFileName(item);
- }
-
- // Otherwise, we want just the top level folder as the game, and the file as everything else
- else
- {
- gamename = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0];
- romname = item.Remove(0, (Path.Combine(basepath, gamename).Length));
- }
- }
-
- // Otherwise, we assume that we have an archive
- else
- {
- // If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
- if (Type == "SuperDAT")
- {
- gamename = parent;
- romname = item.Remove(0, basepath.Length);
- }
-
- // Otherwise, we want the archive name as the game, and the file as everything else
- else
- {
- gamename = parent;
- romname = item.Remove(0, basepath.Length);
- }
- }
-
- // Sanitize the names
- if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
- {
- gamename = gamename.Substring(1);
- }
- if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
- {
- gamename = gamename.Substring(0, gamename.Length - 1);
- }
- if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
- {
- romname = romname.Substring(1);
- }
- if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
- {
- romname = romname.Substring(0, romname.Length - 1);
- }
-
- // Update rom information
- datItem.Name = romname;
- datItem.MachineName = gamename;
- datItem.MachineDescription = gamename;
-
- // Add the file information to the DAT
- lock (Files)
- {
- Files[key].Add(datItem);
- }
-
- logger.User("File added: " + romname + Environment.NewLine);
- }
- catch (IOException ex)
- {
- logger.Error(ex.ToString());
- return;
- }
- }
-
- #endregion
-
- #region Statistics
-
- ///
- /// Recalculate the statistics for the Dat
- ///
- public void RecalculateStats()
- {
- // Wipe out any stats already there
- RomCount = 0;
- DiskCount = 0;
- TotalSize = 0;
- CRCCount = 0;
- MD5Count = 0;
- SHA1Count = 0;
- NodumpCount = 0;
-
- // If we have a blank Dat in any way, return
- if (this == null || Files == null || Files.Count == 0)
- {
- return;
- }
-
- // Loop through and add
- foreach (List roms in Files.Values)
- {
- foreach (Rom rom in roms)
- {
- RomCount += (rom.Type == ItemType.Rom ? 1 : 0);
- DiskCount += (rom.Type == ItemType.Disk ? 1 : 0);
- TotalSize += (rom.ItemStatus == ItemStatus.Nodump ? 0 : rom.Size);
- CRCCount += (String.IsNullOrEmpty(rom.CRC) ? 0 : 1);
- MD5Count += (String.IsNullOrEmpty(rom.MD5) ? 0 : 1);
- SHA1Count += (String.IsNullOrEmpty(rom.SHA1) ? 0 : 1);
- }
- }
- }
-
- ///
- /// Output the stats for the Dat in a human-readable format
- ///
- /// Logger object for file and console writing
- /// True if numbers should be recalculated for the DAT, false otherwise (default)
- /// Number of games to use, -1 means recalculate games (default)
- public void OutputStats(Logger logger, bool recalculate = false, long game = -1)
- {
- // If we're supposed to recalculate the statistics, do so
- if (recalculate)
- {
- RecalculateStats();
- }
-
- SortedDictionary> newroms = DatFile.BucketByGame(Files, false, true, logger, false);
- if (TotalSize < 0)
- {
- TotalSize = Int64.MaxValue + TotalSize;
- }
- logger.User(" Uncompressed size: " + Style.GetBytesReadable(TotalSize) + @"
- Games found: " + (game == -1 ? newroms.Count : game) + @"
- Roms found: " + RomCount + @"
- Disks found: " + DiskCount + @"
- Roms with CRC: " + CRCCount + @"
- Roms with MD5: " + MD5Count + @"
- Roms with SHA-1: " + SHA1Count + @"
- Roms with Nodump status: " + NodumpCount + @"
-");
- }
-
- ///
- /// Output the stats for a list of input dats as files in a human-readable format
- ///
- /// List of input files and folders
- /// True if single DAT stats are output, false otherwise
- /// Logger object for file and console output
- public static void OutputStats(List inputs, bool single, Logger logger)
- {
- // Make sure we have all files
- List newinputs = new List();
- foreach (string input in inputs)
- {
- if (File.Exists(input))
- {
- newinputs.Add(input);
- }
- if (Directory.Exists(input))
- {
- foreach (string file in Directory.GetFiles(input, "*", SearchOption.AllDirectories))
- {
- newinputs.Add(file);
- }
- }
- }
-
- // Init all total variables
- long totalSize = 0;
- long totalGame = 0;
- long totalRom = 0;
- long totalDisk = 0;
- long totalCRC = 0;
- long totalMD5 = 0;
- long totalSHA1 = 0;
- long totalNodump = 0;
-
- /// Now process each of the input files
- foreach (string filename in newinputs)
- {
- logger.Log("Beginning stat collection for '" + filename + "'");
- List games = new List();
- DatFile datdata = new DatFile();
- datdata.Parse(filename, 0, 0, logger);
- SortedDictionary> newroms = BucketByGame(datdata.Files, false, true, logger, false);
-
- // Output single DAT stats (if asked)
- if (single)
- {
- logger.User(@"\nFor file '" + filename + @"':
---------------------------------------------------");
- datdata.OutputStats(logger);
- }
- else
- {
- logger.User("Adding stats for file '" + filename + "'\n");
- }
-
- // Add single DAT stats to totals
- totalSize += datdata.TotalSize;
- totalGame += newroms.Count;
- totalRom += datdata.RomCount;
- totalDisk += datdata.DiskCount;
- totalCRC += datdata.CRCCount;
- totalMD5 += datdata.MD5Count;
- totalSHA1 += datdata.SHA1Count;
- totalNodump += datdata.NodumpCount;
- }
-
- // Output total DAT stats
- if (!single) { logger.User(""); }
- DatFile totaldata = new DatFile
- {
- TotalSize = totalSize,
- RomCount = totalRom,
- DiskCount = totalDisk,
- CRCCount = totalCRC,
- MD5Count = totalMD5,
- SHA1Count = totalSHA1,
- NodumpCount = totalNodump,
- };
- logger.User(@"For ALL DATs found
---------------------------------------------------");
- totaldata.OutputStats(logger, game: totalGame);
- logger.User(@"
-Please check the log folder if the stats scrolled offscreen");
- }
-
- #endregion
-
- #region Bucketing
-
- ///
- /// Take an arbitrarily ordered List and return a Dictionary sorted by Game
- ///
- /// Input unsorted list
- /// True if roms should be deduped, false otherwise
- /// True if games should only be compared on game and file name, false if system and source are counted
- /// Logger object for file and console output
- /// True if the number of hashes counted is to be output (default), false otherwise
- /// SortedDictionary bucketed by game name
- public static SortedDictionary> BucketByGame(List list, bool mergeroms, bool norename, Logger logger, bool output = true)
- {
- Dictionary> dict = new Dictionary>();
- dict.Add("key", list);
- return BucketByGame(dict, mergeroms, norename, logger, output);
- }
-
- ///
- /// Take an arbitrarily bucketed Dictionary and return one sorted by Game
- ///
- /// Input unsorted dictionary
- /// True if roms should be deduped, false otherwise
- /// True if games should only be compared on game and file name, false if system and source are counted
- /// Logger object for file and console output
- /// True if the number of hashes counted is to be output (default), false otherwise
- /// SortedDictionary bucketed by game name
- public static SortedDictionary> BucketByGame(IDictionary> dict, bool mergeroms, bool norename, Logger logger, bool output = true)
- {
- logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output");
-
- SortedDictionary> sortable = new SortedDictionary>();
- long count = 0;
-
- // If we have a null dict or an empty one, output a new dictionary
- if (dict == null || dict.Count == 0)
- {
- return sortable;
- }
-
- // Process each all of the roms
- List keys = dict.Keys.ToList();
- foreach (string key in keys)
- {
- List roms = dict[key];
-
- // If we're merging the roms, do so
- if (mergeroms)
- {
- roms = DatItem.Merge(roms, logger);
- }
-
- // Now add each of the roms to their respective games
- foreach (DatItem rom in roms)
- {
- count++;
- string newkey = (norename ? ""
- : rom.SystemID.ToString().PadLeft(10, '0')
- + "-"
- + rom.SourceID.ToString().PadLeft(10, '0') + "-")
- + (String.IsNullOrEmpty(rom.MachineName)
- ? "Default"
- : rom.MachineName.ToLowerInvariant());
- newkey = HttpUtility.HtmlEncode(newkey);
- if (sortable.ContainsKey(newkey))
- {
- sortable[newkey].Add(rom);
- }
- else
- {
- List temp = new List();
- temp.Add(rom);
- sortable.Add(newkey, temp);
- }
- }
- }
-
- // Now go through and sort all of the lists
- keys = sortable.Keys.ToList();
- foreach (string key in keys)
- {
- List sortedlist = sortable[key];
- DatItem.Sort(ref sortedlist, norename);
- sortable[key] = sortedlist;
- }
-
- // Output the count if told to
- if (output)
- {
- logger.User("A total of " + count + " file hashes will be written out to file");
- }
-
- return sortable;
- }
-
- #endregion
-
- #region Converting and Updating
-
- ///
- /// Convert, update, and filter a DAT file
- ///
- /// Names of the input files and/or folders
- /// User specified inputs contained in a DatData object
- /// Non-zero flag for output format, zero otherwise for default
- /// Optional param for output directory
- /// True if input files should be merged into a single file, false otherwise
- /// Non-zero flag for diffing mode, zero otherwise
- /// True if the diffed files should be cascade diffed, false if diffed files should be reverse cascaded, null otherwise
- /// True if the cascade-diffed files should overwrite their inputs, false otherwise
- /// True if the first cascaded diff file should be skipped on output, false otherwise
- /// True if the date should not be appended to the default name, false otherwise [OBSOLETE]
- /// True to clean the game names to WoD standard, false otherwise (default)
- /// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)
- /// Name of the game to match (can use asterisk-partials)
- /// Name of the rom to match (can use asterisk-partials)
- /// Type of the rom to match
- /// Find roms greater than or equal to this size
- /// Find roms less than or equal to this size
- /// Find roms equal to this size
- /// CRC of the rom to match (can use asterisk-partials)
- /// MD5 of the rom to match (can use asterisk-partials)
- /// SHA-1 of the rom to match (can use asterisk-partials)
- /// Select roms with the given status
- /// True if we are supposed to trim names to NTFS length, false otherwise
- /// True if all games should be replaced by '!', false otherwise
- /// String representing root directory to compare against for length calculation
- /// Integer representing the maximum amount of parallelization to be used
- /// Logging object for console and file output
- public static void Update(List inputFileNames, DatFile datdata, OutputFormat outputFormat, string outDir, bool merge,
- DiffMode diff, bool? cascade, bool inplace, bool skip, bool bare, bool clean, bool softlist, string gamename, string romname, string romtype,
- long sgt, long slt, long seq, string crc, string md5, string sha1, ItemStatus itemStatus, bool trim, bool single, string root, int maxDegreeOfParallelism,
- Logger logger)
- {
- // If we're in merging or diffing mode, use the full list of inputs
- if (merge || diff != 0)
- {
- // Make sure there are no folders in inputs
- List newInputFileNames = new List();
- foreach (string input in inputFileNames)
- {
- if (Directory.Exists(input))
- {
- foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
- {
- try
- {
- newInputFileNames.Add(Path.GetFullPath(file) + "¬" + Path.GetFullPath(input));
- }
- catch (PathTooLongException)
- {
- logger.Warning("The path for " + file + " was too long");
- }
- catch (Exception ex)
- {
- logger.Error(ex.ToString());
- }
- }
- }
- else if (File.Exists(input))
- {
- try
- {
- newInputFileNames.Add(Path.GetFullPath(input) + "¬" + Path.GetDirectoryName(Path.GetFullPath(input)));
- }
- catch (PathTooLongException)
- {
- logger.Warning("The path for " + input + " was too long");
- }
- catch (Exception ex)
- {
- logger.Error(ex.ToString());
- }
- }
- }
-
- // If we're in inverse cascade, reverse the list
- if (cascade == false)
- {
- newInputFileNames.Reverse();
- }
-
- // Create a dictionary of all ROMs from the input DATs
- DatFile userData;
- List datHeaders = PopulateUserData(newInputFileNames, inplace, clean, softlist,
- outDir, datdata, out userData, gamename, romname, romtype, sgt, slt, seq,
- crc, md5, sha1, itemStatus, trim, single, root, maxDegreeOfParallelism, logger);
-
- // Modify the Dictionary if necessary and output the results
- if (diff != 0 && cascade == null)
- {
- userData.DiffNoCascade(diff, outDir, newInputFileNames, logger);
- }
- // If we're in cascade and diff, output only cascaded diffs
- else if (diff != 0 && cascade != null)
- {
- userData.DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip, logger);
- }
- // Output all entries with user-defined merge
- else
- {
- userData.MergeNoDiff(outDir, newInputFileNames, datHeaders, logger);
- }
- }
- // Otherwise, loop through all of the inputs individually
- else
- {
- Parallel.ForEach(inputFileNames,
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- inputFileName =>
- {
- // Clean the input string
- if (inputFileName != "")
- {
- inputFileName = Path.GetFullPath(inputFileName);
- }
-
- if (File.Exists(inputFileName))
- {
- DatFile innerDatdata = (DatFile)datdata.CloneHeader();
- logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
- innerDatdata.Parse(inputFileName, 0, 0, gamename, romname,
- romtype, sgt, slt, seq, crc, md5, sha1, itemStatus, trim, single,
- root, logger, true, clean, softlist, keepext: (innerDatdata.XSV != null));
-
- // If we have roms, output them
- if (innerDatdata.Files.Count != 0)
- {
- innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), logger, overwrite: (outDir != ""));
- }
- }
- else if (Directory.Exists(inputFileName))
- {
- inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
-
- Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories),
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- file =>
- {
- logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
- DatFile innerDatdata = (DatFile)datdata.Clone();
- innerDatdata.Files = null;
- innerDatdata.Parse(file, 0, 0, gamename, romname, romtype, sgt, slt, seq, crc, md5, sha1, itemStatus,
- trim, single, root, logger, true, clean, keepext: (datdata.XSV != null));
-
- // If we have roms, output them
- if (innerDatdata.Files != null && innerDatdata.Files.Count != 0)
- {
- innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), logger, overwrite: (outDir != ""));
- }
- });
- }
- else
- {
- logger.Error("I'm sorry but " + inputFileName + " doesn't exist!");
- }
- });
- }
- return;
- }
-
- ///
- /// Populate the user DatData object from the input files
- ///
- /// Output user DatData object to output
- /// Name of the game to match (can use asterisk-partials)
- /// Name of the rom to match (can use asterisk-partials)
- /// Type of the rom to match
- /// Find roms greater than or equal to this size
- /// Find roms less than or equal to this size
- /// Find roms equal to this size
- /// CRC of the rom to match (can use asterisk-partials)
- /// MD5 of the rom to match (can use asterisk-partials)
- /// SHA-1 of the rom to match (can use asterisk-partials)
- /// Select roms with the given status
- /// True if we are supposed to trim names to NTFS length, false otherwise
- /// True if all games should be replaced by '!', false otherwise
- /// String representing root directory to compare against for length calculation
- /// Integer representing the maximum amount of parallelization to be used
- /// Logging object for console and file output
- /// List of DatData objects representing headers
- private static List PopulateUserData(List inputs, bool inplace, bool clean, bool softlist, string outDir,
- DatFile inputDat, out DatFile userData, string gamename, string romname, string romtype, long sgt, long slt, long seq, string crc,
- string md5, string sha1, ItemStatus itemStatus, bool trim, bool single, string root, int maxDegreeOfParallelism, Logger logger)
- {
- DatFile[] datHeaders = new DatFile[inputs.Count];
- DateTime start = DateTime.Now;
- logger.User("Processing individual DATs");
-
- Parallel.For(0,
- inputs.Count,
- new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
- i =>
- {
- string input = inputs[i];
- logger.User("Adding DAT: " + input.Split('¬')[0]);
- datHeaders[i] = new DatFile
- {
- OutputFormat = (inputDat.OutputFormat != 0 ? inputDat.OutputFormat : 0),
- Files = new Dictionary>(),
- MergeRoms = inputDat.MergeRoms,
- };
-
- datHeaders[i].Parse(input.Split('¬')[0], i, 0, gamename, romname, romtype, sgt, slt, seq,
- crc, md5, sha1, itemStatus, trim, single, root, logger, true, clean, softlist);
- });
-
- logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- logger.User("Populating internal DAT");
- userData = (DatFile)inputDat.CloneHeader();
- userData.Files = new Dictionary>();
- for (int i = 0; i < inputs.Count; i++)
- {
- List keys = datHeaders[i].Files.Keys.ToList();
- foreach (string key in keys)
- {
- if (userData.Files.ContainsKey(key))
- {
- userData.Files[key].AddRange(datHeaders[i].Files[key]);
- }
- else
- {
- userData.Files.Add(key, datHeaders[i].Files[key]);
- }
- datHeaders[i].Files.Remove(key);
- }
- datHeaders[i].Files = null;
- }
-
- logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- return datHeaders.ToList();
- }
-
- ///
- /// Output non-cascading diffs
- ///
- /// Non-zero flag for diffing mode, zero otherwise
- /// Output directory to write the DATs to
- /// List of inputs to write out from
- /// Logging object for console and file output
- public void DiffNoCascade(DiffMode diff, string outDir, List inputs, Logger logger)
- {
- DateTime start = DateTime.Now;
- logger.User("Initializing all output DATs");
-
- // Default vars for use
- string post = "";
- DatFile outerDiffData = new DatFile();
- DatFile dupeData = new DatFile();
-
- // Don't have External dupes
- if ((diff & DiffMode.NoDupes) != 0)
- {
- post = " (No Duplicates)";
- outerDiffData = (DatFile)CloneHeader();
- outerDiffData.FileName += post;
- outerDiffData.Name += post;
- outerDiffData.Description += post;
- outerDiffData.Files = new Dictionary>();
- }
-
- // Have External dupes
- if ((diff & DiffMode.Dupes) != 0)
- {
- post = " (Duplicates)";
- dupeData = (DatFile)CloneHeader();
- dupeData.FileName += post;
- dupeData.Name += post;
- dupeData.Description += post;
- dupeData.Files = new Dictionary>();
- }
-
- // Create a list of DatData objects representing individual output files
- List outDats = new List();
-
- // Loop through each of the inputs and get or create a new DatData object
- if ((diff & DiffMode.Individuals) != 0)
- {
- DatFile[] outDatsArray = new DatFile[inputs.Count];
-
- Parallel.For(0, inputs.Count, j =>
- {
- string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
- DatFile diffData = (DatFile)CloneHeader();
- diffData.FileName += innerpost;
- diffData.Name += innerpost;
- diffData.Description += innerpost;
- diffData.Files = new Dictionary>();
- outDatsArray[j] = diffData;
- });
-
- outDats = outDatsArray.ToList();
- }
- logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- // Now, loop through the dictionary and populate the correct DATs
- start = DateTime.Now;
- logger.User("Populating all output DATs");
- List keys = Files.Keys.ToList();
- foreach (string key in keys)
- {
- List roms = DatItem.Merge(Files[key], logger);
-
- if (roms != null && roms.Count > 0)
- {
- foreach (DatItem rom in roms)
- {
- // No duplicates
- if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
- {
- if (rom.Dupe < DupeType.ExternalHash)
- {
- // Individual DATs that are output
- if ((diff & DiffMode.Individuals) != 0)
- {
- if (outDats[rom.SystemID].Files.ContainsKey(key))
- {
- outDats[rom.SystemID].Files[key].Add(rom);
- }
- else
- {
- List tl = new List();
- tl.Add(rom);
- outDats[rom.SystemID].Files.Add(key, tl);
- }
- }
-
- // Merged no-duplicates DAT
- if ((diff & DiffMode.NoDupes) != 0)
- {
- DatItem newrom = rom;
- newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
-
- if (outerDiffData.Files.ContainsKey(key))
- {
- outerDiffData.Files[key].Add(newrom);
- }
- else
- {
- List tl = new List();
- tl.Add(rom);
- outerDiffData.Files.Add(key, tl);
- }
- }
- }
- }
-
- // Duplicates only
- if ((diff & DiffMode.Dupes) != 0)
- {
- if (rom.Dupe >= DupeType.ExternalHash)
- {
- DatItem newrom = rom;
- newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
-
- if (dupeData.Files.ContainsKey(key))
- {
- dupeData.Files[key].Add(newrom);
- }
- else
- {
- List tl = new List();
- tl.Add(rom);
- dupeData.Files.Add(key, tl);
- }
- }
- }
- }
- }
- }
- logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- // Finally, loop through and output each of the DATs
- start = DateTime.Now;
- logger.User("Outputting all created DATs");
-
- // Output the difflist (a-b)+(b-a) diff
- if ((diff & DiffMode.NoDupes) != 0)
- {
- outerDiffData.WriteToFile(outDir, logger);
- }
-
- // Output the (ab) diff
- if ((diff & DiffMode.Dupes) != 0)
- {
- dupeData.WriteToFile(outDir, logger);
- }
-
- // Output the individual (a-b) DATs
- if ((diff & DiffMode.Individuals) != 0)
- {
- for (int j = 0; j < inputs.Count; j++)
- {
- // If we have an output directory set, replace the path
- string path = outDir + (Path.GetDirectoryName(inputs[j].Split('¬')[0]).Remove(0, inputs[j].Split('¬')[1].Length));
-
- // If we have more than 0 roms, output
- if (outDats[j].Files.Count > 0)
- {
- outDats[j].WriteToFile(path, logger);
- }
- }
- }
- logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
- }
-
- ///
- /// Output cascading diffs
- ///
- /// Output directory to write the DATs to
- /// True if cascaded diffs are outputted in-place, false otherwise
- /// List of inputs to write out from
- /// Dat headers used optionally
- /// True if the first cascaded diff file should be skipped on output, false otherwise
- /// Logging object for console and file output
- public void DiffCascade(string outDir, bool inplace, List inputs, List datHeaders, bool skip, Logger logger)
- {
- string post = "";
-
- // Create a list of DatData objects representing output files
- List outDats = new List();
-
- // Loop through each of the inputs and get or create a new DatData object
- DateTime start = DateTime.Now;
- logger.User("Initializing all output DATs");
-
- DatFile[] outDatsArray = new DatFile[inputs.Count];
-
- Parallel.For(0, inputs.Count, j =>
- {
- string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
- DatFile diffData;
-
- // If we're in inplace mode, take the appropriate DatData object already stored
- if (inplace || !String.IsNullOrEmpty(outDir))
- {
- diffData = datHeaders[j];
- }
- else
- {
- diffData = (DatFile)CloneHeader();
- diffData.FileName += post;
- diffData.Name += post;
- diffData.Description += post;
- }
- diffData.Files = new Dictionary>();
-
- outDatsArray[j] = diffData;
- });
-
- outDats = outDatsArray.ToList();
- logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- // Now, loop through the dictionary and populate the correct DATs
- start = DateTime.Now;
- logger.User("Populating all output DATs");
- List keys = Files.Keys.ToList();
-
- foreach (string key in keys)
- {
- List roms = DatItem.Merge(Files[key], logger);
-
- if (roms != null && roms.Count > 0)
- {
- foreach (DatItem rom in roms)
- {
- // There's odd cases where there are items with System ID < 0. Skip them for now
- if (rom.SystemID < 0)
- {
- logger.Warning("Item found with a <0 SystemID: " + rom.Name);
- continue;
- }
-
- if (outDats[rom.SystemID].Files.ContainsKey(key))
- {
- outDats[rom.SystemID].Files[key].Add(rom);
- }
- else
- {
- List tl = new List();
- tl.Add(rom);
- outDats[rom.SystemID].Files.Add(key, tl);
- }
- }
- }
- }
- logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
-
- // Finally, loop through and output each of the DATs
- start = DateTime.Now;
- logger.User("Outputting all created DATs");
- for (int j = (skip ? 1 : 0); j < inputs.Count; j++)
- {
- // If we have an output directory set, replace the path
- string path = "";
- if (inplace)
- {
- path = Path.GetDirectoryName(inputs[j].Split('¬')[0]);
- }
- else if (!String.IsNullOrEmpty(outDir))
- {
- path = outDir + (Path.GetDirectoryName(inputs[j].Split('¬')[0]).Remove(0, inputs[j].Split('¬')[1].Length));
- }
-
- // If we have more than 0 roms, output
- if (outDats[j].Files.Count > 0)
- {
- outDats[j].WriteToFile(path, logger);
- }
- }
- logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
- }
-
- ///
- /// Output user defined merge
- ///
- /// Output directory to write the DATs to
- /// List of inputs to write out from
- /// Dat headers used optionally
- /// Logging object for console and file output
- public void MergeNoDiff(string outDir, List inputs, List datHeaders, Logger logger)
- {
- // If we're in SuperDAT mode, prefix all games with their respective DATs
- if (Type == "SuperDAT")
- {
- List keys = Files.Keys.ToList();
- foreach (string key in keys)
- {
- List newroms = new List();
- foreach (DatItem rom in Files[key])
- {
- DatItem newrom = rom;
- string filename = inputs[newrom.SystemID].Split('¬')[0];
- string rootpath = inputs[newrom.SystemID].Split('¬')[1];
-
- rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString());
- filename = filename.Remove(0, rootpath.Length);
- newrom.MachineName = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
- + Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
- + newrom.MachineName;
- newroms.Add(newrom);
- }
- Files[key] = newroms;
- }
- }
-
- // Output a DAT only if there are roms
- if (Files.Count != 0)
- {
- WriteToFile(outDir, logger);
- }
- }
-
- #endregion
-
#region DAT Writing
///
@@ -4522,6 +3254,1149 @@ Please check the log folder if the stats scrolled offscreen");
#endregion
+ #region Populate DAT from Directory
+
+ ///
+ /// Create a new Dat from a directory
+ ///
+ /// Base folder to be used in creating the DAT
+ /// True if MD5 hashes should be skipped over, false otherwise
+ /// True if SHA-1 hashes should be skipped over, false otherwise
+ /// True if the date should be omitted from the DAT, false otherwise
+ /// True if archives should be treated as files, false otherwise
+ /// True if GZIP archives should be treated as files, false otherwise
+ /// True if blank items should be created for empty folders, false otherwise
+ /// True if dates should be archived for all files, false otherwise
+ /// Name of the directory to create a temp folder in (blank is current directory)
+ /// True if files should be copied to the temp directory before hashing, false otherwise
+ /// Integer representing the maximum amount of parallelization to be used
+ /// Logger object for console and file output
+ public bool PopulateDatFromDir(string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
+ bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
+ {
+ // If the description is defined but not the name, set the name from the description
+ if (String.IsNullOrEmpty(Name) && !String.IsNullOrEmpty(Description))
+ {
+ Name = Description;
+ }
+
+ // If the name is defined but not the description, set the description from the name
+ else if (!String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
+ {
+ Description = Name + (bare ? "" : " (" + Date + ")");
+ }
+
+ // If neither the name or description are defined, set them from the automatic values
+ else if (String.IsNullOrEmpty(Name) && String.IsNullOrEmpty(Description))
+ {
+ Name = basePath.Split(Path.DirectorySeparatorChar).Last();
+ Description = Name + (bare ? "" : " (" + Date + ")");
+ }
+
+ // Process the input folder
+ logger.Log("Folder found: " + basePath);
+
+ // Process the files in all subfolders
+ List files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList();
+ Parallel.ForEach(files,
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ item =>
+ {
+ DFDProcessPossibleArchive(item, basePath, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, addBlanks, addDate,
+ tempDir, copyFiles, maxDegreeOfParallelism, logger);
+ });
+
+ // Now find all folders that are empty, if we are supposed to
+ if (!Romba && addBlanks)
+ {
+ List empties = Directory.EnumerateDirectories(basePath, "*", SearchOption.AllDirectories).ToList();
+ Parallel.ForEach(empties,
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ dir =>
+ {
+ if (Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly).Count() == 0)
+ {
+ // Get the full path for the directory
+ string fulldir = Path.GetFullPath(dir);
+
+ // Set the temporary variables
+ string gamename = "";
+ string romname = "";
+
+ // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
+ if (Type == "SuperDAT")
+ {
+ gamename = fulldir.Remove(0, basePath.Length + 1);
+ romname = "-";
+ }
+
+ // Otherwise, we want just the top level folder as the game, and the file as everything else
+ else
+ {
+ gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0];
+ romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "-");
+ }
+
+ // Sanitize the names
+ if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ gamename = gamename.Substring(1);
+ }
+ if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ gamename = gamename.Substring(0, gamename.Length - 1);
+ }
+ if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ romname = romname.Substring(1);
+ }
+ if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ romname = romname.Substring(0, romname.Length - 1);
+ }
+
+ logger.Log("Adding blank empty folder: " + gamename);
+ Files["null"].Add(new Rom(romname, gamename));
+ }
+ });
+ }
+
+ // Now that we're done, delete the temp folder (if it's not the default)
+ logger.User("Cleaning temp folder");
+ try
+ {
+ if (tempDir != Path.GetTempPath())
+ {
+ Directory.Delete(tempDir, true);
+ }
+ }
+ catch
+ {
+ // Just absorb the error for now
+ }
+
+ return true;
+ }
+
+ ///
+ /// Check a given file for hashes, based on current settings
+ ///
+ /// Filename of the item to be checked
+ /// Base folder to be used in creating the DAT
+ /// True if MD5 hashes should be skipped over, false otherwise
+ /// True if SHA-1 hashes should be skipped over, false otherwise
+ /// True if the date should be omitted from the DAT, false otherwise
+ /// True if archives should be treated as files, false otherwise
+ /// True if GZIP archives should be treated as files, false otherwise
+ /// True if blank items should be created for empty folders, false otherwise
+ /// True if dates should be archived for all files, false otherwise
+ /// Name of the directory to create a temp folder in (blank is current directory)
+ /// True if files should be copied to the temp directory before hashing, false otherwise
+ /// Integer representing the maximum amount of parallelization to be used
+ /// Logger object for console and file output
+ private void DFDProcessPossibleArchive(string item, string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
+ bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
+ {
+ // Define the temporary directory
+ string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar;
+
+ // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
+ if (Romba)
+ {
+ Rom rom = FileTools.GetTorrentGZFileInfo(item, logger);
+
+ // If the rom is valid, write it out
+ if (rom.Name != null)
+ {
+ // Add the list if it doesn't exist already
+ string key = rom.Size + "-" + rom.CRC;
+
+ lock (Files)
+ {
+ if (!Files.ContainsKey(key))
+ {
+ Files.Add(key, new List());
+ }
+
+ Files[key].Add(rom);
+ logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+ }
+
+ }
+ else
+ {
+ logger.User("File not added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+ return;
+ }
+
+ return;
+ }
+
+ // If we're copying files, copy it first and get the new filename
+ string newItem = item;
+ string newBasePath = basePath;
+ if (copyFiles)
+ {
+ newBasePath = Path.Combine(tempDir, Path.GetRandomFileName());
+ newItem = Path.GetFullPath(Path.Combine(newBasePath, Path.GetFullPath(item).Remove(0, basePath.Length + 1)));
+ Directory.CreateDirectory(Path.GetDirectoryName(newItem));
+ File.Copy(item, newItem, true);
+ }
+
+ // If both deep hash skip flags are set, do a quickscan
+ if (noMD5 && noSHA1)
+ {
+ ArchiveType? type = FileTools.GetCurrentArchiveType(newItem, logger);
+
+ // If we have an archive, scan it
+ if (type != null && !archivesAsFiles)
+ {
+ List extracted = FileTools.GetArchiveFileInfo(newItem, logger);
+
+ foreach (Rom rom in extracted)
+ {
+ DFDProcessFileHelper(newItem,
+ rom,
+ basePath,
+ (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item),
+ logger);
+ }
+ }
+ // Otherwise, just get the info on the file itself
+ else if (File.Exists(newItem))
+ {
+ DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
+ }
+ }
+ // Otherwise, attempt to extract the files to the temporary directory
+ else
+ {
+ bool encounteredErrors = FileTools.ExtractArchive(newItem,
+ tempSubDir,
+ (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ (!archivesAsFiles && enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External),
+ (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ (archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ logger);
+
+ // If the file was an archive and was extracted successfully, check it
+ if (!encounteredErrors)
+ {
+ logger.Log(Path.GetFileName(item) + " treated like an archive");
+ List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList();
+ Parallel.ForEach(extracted,
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ entry =>
+ {
+ DFDProcessFile(entry,
+ Path.Combine((Type == "SuperDAT"
+ ? (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length)
+ : ""),
+ Path.GetFileNameWithoutExtension(item)),
+ tempSubDir,
+ noMD5,
+ noSHA1,
+ addDate,
+ logger);
+ });
+ }
+ // Otherwise, just get the info on the file itself
+ else if (File.Exists(newItem))
+ {
+ DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
+ }
+ }
+
+ // Cue to delete the file if it's a copy
+ if (copyFiles && item != newItem)
+ {
+ try
+ {
+ Directory.Delete(newBasePath, true);
+ }
+ catch { }
+ }
+
+ // Delete the sub temp directory
+ if (Directory.Exists(tempSubDir))
+ {
+ Directory.Delete(tempSubDir, true);
+ }
+ }
+
+ ///
+ /// Process a single file as a file
+ ///
+ /// File to be added
+ /// Path the represents the parent directory
+ /// Parent game to be used
+ private void DFDProcessFile(string item, string parent, string basePath, bool noMD5, bool noSHA1, bool addDate, Logger logger)
+ {
+ logger.Log(Path.GetFileName(item) + " treated like a file");
+ Rom rom = FileTools.GetSingleFileInfo(item, noMD5: noMD5, noSHA1: noSHA1, date: addDate);
+
+ DFDProcessFileHelper(item, rom, basePath, parent, logger);
+ }
+
+ ///
+ /// Process a single file as a file (with found Rom data)
+ ///
+ /// File to be added
+ /// Rom data to be used to write to file
+ /// Path the represents the parent directory
+ /// Parent game to be used
+ private void DFDProcessFileHelper(string item, DatItem datItem, string basepath, string parent, Logger logger)
+ {
+ // If the datItem isn't a Rom or Disk, return
+ if (datItem.Type != ItemType.Rom && datItem.Type != ItemType.Disk)
+ {
+ return;
+ }
+
+ string key = "";
+ if (datItem.Type == ItemType.Rom)
+ {
+ key = ((Rom)datItem).Size + "-" + ((Rom)datItem).CRC;
+ }
+ else
+ {
+ key = ((Disk)datItem).MD5;
+ }
+
+ // Add the list if it doesn't exist already
+ lock (Files)
+ {
+ if (!Files.ContainsKey(key))
+ {
+ Files.Add(key, new List());
+ }
+ }
+
+ try
+ {
+ // If the basepath ends with a directory separator, remove it
+ if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ basepath += Path.DirectorySeparatorChar.ToString();
+ }
+
+ // Make sure we have the full item path
+ item = Path.GetFullPath(item);
+
+ // Get the data to be added as game and item names
+ string gamename = "";
+ string romname = "";
+
+ // If the parent is blank, then we have a non-archive file
+ if (parent == "")
+ {
+ // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
+ if (Type == "SuperDAT")
+ {
+ gamename = Path.GetDirectoryName(item.Remove(0, basepath.Length));
+ romname = Path.GetFileName(item);
+ }
+
+ // Otherwise, we want just the top level folder as the game, and the file as everything else
+ else
+ {
+ gamename = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0];
+ romname = item.Remove(0, (Path.Combine(basepath, gamename).Length));
+ }
+ }
+
+ // Otherwise, we assume that we have an archive
+ else
+ {
+ // If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
+ if (Type == "SuperDAT")
+ {
+ gamename = parent;
+ romname = item.Remove(0, basepath.Length);
+ }
+
+ // Otherwise, we want the archive name as the game, and the file as everything else
+ else
+ {
+ gamename = parent;
+ romname = item.Remove(0, basepath.Length);
+ }
+ }
+
+ // Sanitize the names
+ if (gamename.StartsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ gamename = gamename.Substring(1);
+ }
+ if (gamename.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ gamename = gamename.Substring(0, gamename.Length - 1);
+ }
+ if (romname.StartsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ romname = romname.Substring(1);
+ }
+ if (romname.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ romname = romname.Substring(0, romname.Length - 1);
+ }
+
+ // Update rom information
+ datItem.Name = romname;
+ datItem.MachineName = gamename;
+ datItem.MachineDescription = gamename;
+
+ // Add the file information to the DAT
+ lock (Files)
+ {
+ Files[key].Add(datItem);
+ }
+
+ logger.User("File added: " + romname + Environment.NewLine);
+ }
+ catch (IOException ex)
+ {
+ logger.Error(ex.ToString());
+ return;
+ }
+ }
+
+ #endregion
+
+ #region Statistics
+
+ ///
+ /// Recalculate the statistics for the Dat
+ ///
+ public void RecalculateStats()
+ {
+ // Wipe out any stats already there
+ RomCount = 0;
+ DiskCount = 0;
+ TotalSize = 0;
+ CRCCount = 0;
+ MD5Count = 0;
+ SHA1Count = 0;
+ NodumpCount = 0;
+
+ // If we have a blank Dat in any way, return
+ if (this == null || Files == null || Files.Count == 0)
+ {
+ return;
+ }
+
+ // Loop through and add
+ foreach (List roms in Files.Values)
+ {
+ foreach (Rom rom in roms)
+ {
+ RomCount += (rom.Type == ItemType.Rom ? 1 : 0);
+ DiskCount += (rom.Type == ItemType.Disk ? 1 : 0);
+ TotalSize += (rom.ItemStatus == ItemStatus.Nodump ? 0 : rom.Size);
+ CRCCount += (String.IsNullOrEmpty(rom.CRC) ? 0 : 1);
+ MD5Count += (String.IsNullOrEmpty(rom.MD5) ? 0 : 1);
+ SHA1Count += (String.IsNullOrEmpty(rom.SHA1) ? 0 : 1);
+ }
+ }
+ }
+
+ ///
+ /// Output the stats for the Dat in a human-readable format
+ ///
+ /// Logger object for file and console writing
+ /// True if numbers should be recalculated for the DAT, false otherwise (default)
+ /// Number of games to use, -1 means recalculate games (default)
+ public void OutputStats(Logger logger, bool recalculate = false, long game = -1)
+ {
+ // If we're supposed to recalculate the statistics, do so
+ if (recalculate)
+ {
+ RecalculateStats();
+ }
+
+ SortedDictionary> newroms = DatFile.BucketByGame(Files, false, true, logger, false);
+ if (TotalSize < 0)
+ {
+ TotalSize = Int64.MaxValue + TotalSize;
+ }
+ logger.User(" Uncompressed size: " + Style.GetBytesReadable(TotalSize) + @"
+ Games found: " + (game == -1 ? newroms.Count : game) + @"
+ Roms found: " + RomCount + @"
+ Disks found: " + DiskCount + @"
+ Roms with CRC: " + CRCCount + @"
+ Roms with MD5: " + MD5Count + @"
+ Roms with SHA-1: " + SHA1Count + @"
+ Roms with Nodump status: " + NodumpCount + @"
+");
+ }
+
+ #endregion
+
+ #region Converting and Updating
+
+ ///
+ /// Output non-cascading diffs
+ ///
+ /// Non-zero flag for diffing mode, zero otherwise
+ /// Output directory to write the DATs to
+ /// List of inputs to write out from
+ /// Logging object for console and file output
+ public void DiffNoCascade(DiffMode diff, string outDir, List inputs, Logger logger)
+ {
+ DateTime start = DateTime.Now;
+ logger.User("Initializing all output DATs");
+
+ // Default vars for use
+ string post = "";
+ DatFile outerDiffData = new DatFile();
+ DatFile dupeData = new DatFile();
+
+ // Don't have External dupes
+ if ((diff & DiffMode.NoDupes) != 0)
+ {
+ post = " (No Duplicates)";
+ outerDiffData = (DatFile)CloneHeader();
+ outerDiffData.FileName += post;
+ outerDiffData.Name += post;
+ outerDiffData.Description += post;
+ outerDiffData.Files = new Dictionary>();
+ }
+
+ // Have External dupes
+ if ((diff & DiffMode.Dupes) != 0)
+ {
+ post = " (Duplicates)";
+ dupeData = (DatFile)CloneHeader();
+ dupeData.FileName += post;
+ dupeData.Name += post;
+ dupeData.Description += post;
+ dupeData.Files = new Dictionary>();
+ }
+
+ // Create a list of DatData objects representing individual output files
+ List outDats = new List();
+
+ // Loop through each of the inputs and get or create a new DatData object
+ if ((diff & DiffMode.Individuals) != 0)
+ {
+ DatFile[] outDatsArray = new DatFile[inputs.Count];
+
+ Parallel.For(0, inputs.Count, j =>
+ {
+ string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
+ DatFile diffData = (DatFile)CloneHeader();
+ diffData.FileName += innerpost;
+ diffData.Name += innerpost;
+ diffData.Description += innerpost;
+ diffData.Files = new Dictionary>();
+ outDatsArray[j] = diffData;
+ });
+
+ outDats = outDatsArray.ToList();
+ }
+ logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ // Now, loop through the dictionary and populate the correct DATs
+ start = DateTime.Now;
+ logger.User("Populating all output DATs");
+ List keys = Files.Keys.ToList();
+ foreach (string key in keys)
+ {
+ List roms = DatItem.Merge(Files[key], logger);
+
+ if (roms != null && roms.Count > 0)
+ {
+ foreach (DatItem rom in roms)
+ {
+ // No duplicates
+ if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
+ {
+ if (rom.Dupe < DupeType.ExternalHash)
+ {
+ // Individual DATs that are output
+ if ((diff & DiffMode.Individuals) != 0)
+ {
+ if (outDats[rom.SystemID].Files.ContainsKey(key))
+ {
+ outDats[rom.SystemID].Files[key].Add(rom);
+ }
+ else
+ {
+ List tl = new List();
+ tl.Add(rom);
+ outDats[rom.SystemID].Files.Add(key, tl);
+ }
+ }
+
+ // Merged no-duplicates DAT
+ if ((diff & DiffMode.NoDupes) != 0)
+ {
+ DatItem newrom = rom;
+ newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
+
+ if (outerDiffData.Files.ContainsKey(key))
+ {
+ outerDiffData.Files[key].Add(newrom);
+ }
+ else
+ {
+ List tl = new List();
+ tl.Add(rom);
+ outerDiffData.Files.Add(key, tl);
+ }
+ }
+ }
+ }
+
+ // Duplicates only
+ if ((diff & DiffMode.Dupes) != 0)
+ {
+ if (rom.Dupe >= DupeType.ExternalHash)
+ {
+ DatItem newrom = rom;
+ newrom.MachineName += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
+
+ if (dupeData.Files.ContainsKey(key))
+ {
+ dupeData.Files[key].Add(newrom);
+ }
+ else
+ {
+ List tl = new List();
+ tl.Add(rom);
+ dupeData.Files.Add(key, tl);
+ }
+ }
+ }
+ }
+ }
+ }
+ logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ // Finally, loop through and output each of the DATs
+ start = DateTime.Now;
+ logger.User("Outputting all created DATs");
+
+ // Output the difflist (a-b)+(b-a) diff
+ if ((diff & DiffMode.NoDupes) != 0)
+ {
+ outerDiffData.WriteToFile(outDir, logger);
+ }
+
+ // Output the (ab) diff
+ if ((diff & DiffMode.Dupes) != 0)
+ {
+ dupeData.WriteToFile(outDir, logger);
+ }
+
+ // Output the individual (a-b) DATs
+ if ((diff & DiffMode.Individuals) != 0)
+ {
+ for (int j = 0; j < inputs.Count; j++)
+ {
+ // If we have an output directory set, replace the path
+ string path = outDir + (Path.GetDirectoryName(inputs[j].Split('¬')[0]).Remove(0, inputs[j].Split('¬')[1].Length));
+
+ // If we have more than 0 roms, output
+ if (outDats[j].Files.Count > 0)
+ {
+ outDats[j].WriteToFile(path, logger);
+ }
+ }
+ }
+ logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+ }
+
+ ///
+ /// Output cascading diffs
+ ///
+ /// Output directory to write the DATs to
+ /// True if cascaded diffs are outputted in-place, false otherwise
+ /// List of inputs to write out from
+ /// Dat headers used optionally
+ /// True if the first cascaded diff file should be skipped on output, false otherwise
+ /// Logging object for console and file output
+ public void DiffCascade(string outDir, bool inplace, List inputs, List datHeaders, bool skip, Logger logger)
+ {
+ string post = "";
+
+ // Create a list of DatData objects representing output files
+ List outDats = new List();
+
+ // Loop through each of the inputs and get or create a new DatData object
+ DateTime start = DateTime.Now;
+ logger.User("Initializing all output DATs");
+
+ DatFile[] outDatsArray = new DatFile[inputs.Count];
+
+ Parallel.For(0, inputs.Count, j =>
+ {
+ string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
+ DatFile diffData;
+
+ // If we're in inplace mode, take the appropriate DatData object already stored
+ if (inplace || !String.IsNullOrEmpty(outDir))
+ {
+ diffData = datHeaders[j];
+ }
+ else
+ {
+ diffData = (DatFile)CloneHeader();
+ diffData.FileName += post;
+ diffData.Name += post;
+ diffData.Description += post;
+ }
+ diffData.Files = new Dictionary>();
+
+ outDatsArray[j] = diffData;
+ });
+
+ outDats = outDatsArray.ToList();
+ logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ // Now, loop through the dictionary and populate the correct DATs
+ start = DateTime.Now;
+ logger.User("Populating all output DATs");
+ List keys = Files.Keys.ToList();
+
+ foreach (string key in keys)
+ {
+ List roms = DatItem.Merge(Files[key], logger);
+
+ if (roms != null && roms.Count > 0)
+ {
+ foreach (DatItem rom in roms)
+ {
+ // There's odd cases where there are items with System ID < 0. Skip them for now
+ if (rom.SystemID < 0)
+ {
+ logger.Warning("Item found with a <0 SystemID: " + rom.Name);
+ continue;
+ }
+
+ if (outDats[rom.SystemID].Files.ContainsKey(key))
+ {
+ outDats[rom.SystemID].Files[key].Add(rom);
+ }
+ else
+ {
+ List tl = new List();
+ tl.Add(rom);
+ outDats[rom.SystemID].Files.Add(key, tl);
+ }
+ }
+ }
+ }
+ logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ // Finally, loop through and output each of the DATs
+ start = DateTime.Now;
+ logger.User("Outputting all created DATs");
+ for (int j = (skip ? 1 : 0); j < inputs.Count; j++)
+ {
+ // If we have an output directory set, replace the path
+ string path = "";
+ if (inplace)
+ {
+ path = Path.GetDirectoryName(inputs[j].Split('¬')[0]);
+ }
+ else if (!String.IsNullOrEmpty(outDir))
+ {
+ path = outDir + (Path.GetDirectoryName(inputs[j].Split('¬')[0]).Remove(0, inputs[j].Split('¬')[1].Length));
+ }
+
+ // If we have more than 0 roms, output
+ if (outDats[j].Files.Count > 0)
+ {
+ outDats[j].WriteToFile(path, logger);
+ }
+ }
+ logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+ }
+
+ ///
+ /// Output user defined merge
+ ///
+ /// Output directory to write the DATs to
+ /// List of inputs to write out from
+ /// Dat headers used optionally
+ /// Logging object for console and file output
+ public void MergeNoDiff(string outDir, List inputs, List datHeaders, Logger logger)
+ {
+ // If we're in SuperDAT mode, prefix all games with their respective DATs
+ if (Type == "SuperDAT")
+ {
+ List keys = Files.Keys.ToList();
+ foreach (string key in keys)
+ {
+ List newroms = new List();
+ foreach (DatItem rom in Files[key])
+ {
+ DatItem newrom = rom;
+ string filename = inputs[newrom.SystemID].Split('¬')[0];
+ string rootpath = inputs[newrom.SystemID].Split('¬')[1];
+
+ rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString());
+ filename = filename.Remove(0, rootpath.Length);
+ newrom.MachineName = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+ + Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+ + newrom.MachineName;
+ newroms.Add(newrom);
+ }
+ Files[key] = newroms;
+ }
+ }
+
+ // Output a DAT only if there are roms
+ if (Files.Count != 0)
+ {
+ WriteToFile(outDir, logger);
+ }
+ }
+
+ #endregion
+
+ #endregion // Instance Methods
+
+ #region Static Methods
+
+ #region Bucketing
+
+ ///
+ /// Take an arbitrarily ordered List and return a Dictionary sorted by Game
+ ///
+ /// Input unsorted list
+ /// True if roms should be deduped, false otherwise
+ /// True if games should only be compared on game and file name, false if system and source are counted
+ /// Logger object for file and console output
+ /// True if the number of hashes counted is to be output (default), false otherwise
+ /// SortedDictionary bucketed by game name
+ public static SortedDictionary> BucketByGame(List list, bool mergeroms, bool norename, Logger logger, bool output = true)
+ {
+ Dictionary> dict = new Dictionary>();
+ dict.Add("key", list);
+ return BucketByGame(dict, mergeroms, norename, logger, output);
+ }
+
+ ///
+ /// Take an arbitrarily bucketed Dictionary and return one sorted by Game
+ ///
+ /// Input unsorted dictionary
+ /// True if roms should be deduped, false otherwise
+ /// True if games should only be compared on game and file name, false if system and source are counted
+ /// Logger object for file and console output
+ /// True if the number of hashes counted is to be output (default), false otherwise
+ /// SortedDictionary bucketed by game name
+ public static SortedDictionary> BucketByGame(IDictionary> dict, bool mergeroms, bool norename, Logger logger, bool output = true)
+ {
+ logger.User("Organizing " + (mergeroms ? "and merging " : "") + "roms for output");
+
+ SortedDictionary> sortable = new SortedDictionary>();
+ long count = 0;
+
+ // If we have a null dict or an empty one, output a new dictionary
+ if (dict == null || dict.Count == 0)
+ {
+ return sortable;
+ }
+
+ // Process each all of the roms
+ List keys = dict.Keys.ToList();
+ foreach (string key in keys)
+ {
+ List roms = dict[key];
+
+ // If we're merging the roms, do so
+ if (mergeroms)
+ {
+ roms = DatItem.Merge(roms, logger);
+ }
+
+ // Now add each of the roms to their respective games
+ foreach (DatItem rom in roms)
+ {
+ count++;
+ string newkey = (norename ? ""
+ : rom.SystemID.ToString().PadLeft(10, '0')
+ + "-"
+ + rom.SourceID.ToString().PadLeft(10, '0') + "-")
+ + (String.IsNullOrEmpty(rom.MachineName)
+ ? "Default"
+ : rom.MachineName.ToLowerInvariant());
+ newkey = HttpUtility.HtmlEncode(newkey);
+ if (sortable.ContainsKey(newkey))
+ {
+ sortable[newkey].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ sortable.Add(newkey, temp);
+ }
+ }
+ }
+
+ // Now go through and sort all of the lists
+ keys = sortable.Keys.ToList();
+ foreach (string key in keys)
+ {
+ List sortedlist = sortable[key];
+ DatItem.Sort(ref sortedlist, norename);
+ sortable[key] = sortedlist;
+ }
+
+ // Output the count if told to
+ if (output)
+ {
+ logger.User("A total of " + count + " file hashes will be written out to file");
+ }
+
+ return sortable;
+ }
+
+ #endregion
+
+ #region Converting and Updating
+
+ ///
+ /// Convert, update, and filter a DAT file
+ ///
+ /// Names of the input files and/or folders
+ /// User specified inputs contained in a DatData object
+ /// Non-zero flag for output format, zero otherwise for default
+ /// Optional param for output directory
+ /// True if input files should be merged into a single file, false otherwise
+ /// Non-zero flag for diffing mode, zero otherwise
+ /// True if the diffed files should be cascade diffed, false if diffed files should be reverse cascaded, null otherwise
+ /// True if the cascade-diffed files should overwrite their inputs, false otherwise
+ /// True if the first cascaded diff file should be skipped on output, false otherwise
+ /// True if the date should not be appended to the default name, false otherwise [OBSOLETE]
+ /// True to clean the game names to WoD standard, false otherwise (default)
+ /// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)
+ /// Name of the game to match (can use asterisk-partials)
+ /// Name of the rom to match (can use asterisk-partials)
+ /// Type of the rom to match
+ /// Find roms greater than or equal to this size
+ /// Find roms less than or equal to this size
+ /// Find roms equal to this size
+ /// CRC of the rom to match (can use asterisk-partials)
+ /// MD5 of the rom to match (can use asterisk-partials)
+ /// SHA-1 of the rom to match (can use asterisk-partials)
+ /// Select roms with the given status
+ /// True if we are supposed to trim names to NTFS length, false otherwise
+ /// True if all games should be replaced by '!', false otherwise
+ /// String representing root directory to compare against for length calculation
+ /// Integer representing the maximum amount of parallelization to be used
+ /// Logging object for console and file output
+ public static void Update(List inputFileNames, DatFile datdata, OutputFormat outputFormat, string outDir, bool merge,
+ DiffMode diff, bool? cascade, bool inplace, bool skip, bool bare, bool clean, bool softlist, string gamename, string romname, string romtype,
+ long sgt, long slt, long seq, string crc, string md5, string sha1, ItemStatus itemStatus, bool trim, bool single, string root, int maxDegreeOfParallelism,
+ Logger logger)
+ {
+ // If we're in merging or diffing mode, use the full list of inputs
+ if (merge || diff != 0)
+ {
+ // Make sure there are no folders in inputs
+ List newInputFileNames = new List();
+ foreach (string input in inputFileNames)
+ {
+ if (Directory.Exists(input))
+ {
+ foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
+ {
+ try
+ {
+ newInputFileNames.Add(Path.GetFullPath(file) + "¬" + Path.GetFullPath(input));
+ }
+ catch (PathTooLongException)
+ {
+ logger.Warning("The path for " + file + " was too long");
+ }
+ catch (Exception ex)
+ {
+ logger.Error(ex.ToString());
+ }
+ }
+ }
+ else if (File.Exists(input))
+ {
+ try
+ {
+ newInputFileNames.Add(Path.GetFullPath(input) + "¬" + Path.GetDirectoryName(Path.GetFullPath(input)));
+ }
+ catch (PathTooLongException)
+ {
+ logger.Warning("The path for " + input + " was too long");
+ }
+ catch (Exception ex)
+ {
+ logger.Error(ex.ToString());
+ }
+ }
+ }
+
+ // If we're in inverse cascade, reverse the list
+ if (cascade == false)
+ {
+ newInputFileNames.Reverse();
+ }
+
+ // Create a dictionary of all ROMs from the input DATs
+ DatFile userData;
+ List datHeaders = PopulateUserData(newInputFileNames, inplace, clean, softlist,
+ outDir, datdata, out userData, gamename, romname, romtype, sgt, slt, seq,
+ crc, md5, sha1, itemStatus, trim, single, root, maxDegreeOfParallelism, logger);
+
+ // Modify the Dictionary if necessary and output the results
+ if (diff != 0 && cascade == null)
+ {
+ userData.DiffNoCascade(diff, outDir, newInputFileNames, logger);
+ }
+ // If we're in cascade and diff, output only cascaded diffs
+ else if (diff != 0 && cascade != null)
+ {
+ userData.DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip, logger);
+ }
+ // Output all entries with user-defined merge
+ else
+ {
+ userData.MergeNoDiff(outDir, newInputFileNames, datHeaders, logger);
+ }
+ }
+ // Otherwise, loop through all of the inputs individually
+ else
+ {
+ Parallel.ForEach(inputFileNames,
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ inputFileName =>
+ {
+ // Clean the input string
+ if (inputFileName != "")
+ {
+ inputFileName = Path.GetFullPath(inputFileName);
+ }
+
+ if (File.Exists(inputFileName))
+ {
+ DatFile innerDatdata = (DatFile)datdata.CloneHeader();
+ logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
+ innerDatdata.Parse(inputFileName, 0, 0, gamename, romname,
+ romtype, sgt, slt, seq, crc, md5, sha1, itemStatus, trim, single,
+ root, logger, true, clean, softlist, keepext: (innerDatdata.XSV != null));
+
+ // If we have roms, output them
+ if (innerDatdata.Files.Count != 0)
+ {
+ innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), logger, overwrite: (outDir != ""));
+ }
+ }
+ else if (Directory.Exists(inputFileName))
+ {
+ inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
+
+ Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories),
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ file =>
+ {
+ logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
+ DatFile innerDatdata = (DatFile)datdata.Clone();
+ innerDatdata.Files = null;
+ innerDatdata.Parse(file, 0, 0, gamename, romname, romtype, sgt, slt, seq, crc, md5, sha1, itemStatus,
+ trim, single, root, logger, true, clean, keepext: (datdata.XSV != null));
+
+ // If we have roms, output them
+ if (innerDatdata.Files != null && innerDatdata.Files.Count != 0)
+ {
+ innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), logger, overwrite: (outDir != ""));
+ }
+ });
+ }
+ else
+ {
+ logger.Error("I'm sorry but " + inputFileName + " doesn't exist!");
+ }
+ });
+ }
+ return;
+ }
+
+ ///
+ /// Populate the user DatData object from the input files
+ ///
+ /// Output user DatData object to output
+ /// Name of the game to match (can use asterisk-partials)
+ /// Name of the rom to match (can use asterisk-partials)
+ /// Type of the rom to match
+ /// Find roms greater than or equal to this size
+ /// Find roms less than or equal to this size
+ /// Find roms equal to this size
+ /// CRC of the rom to match (can use asterisk-partials)
+ /// MD5 of the rom to match (can use asterisk-partials)
+ /// SHA-1 of the rom to match (can use asterisk-partials)
+ /// Select roms with the given status
+ /// True if we are supposed to trim names to NTFS length, false otherwise
+ /// True if all games should be replaced by '!', false otherwise
+ /// String representing root directory to compare against for length calculation
+ /// Integer representing the maximum amount of parallelization to be used
+ /// Logging object for console and file output
+ /// List of DatData objects representing headers
+ private static List PopulateUserData(List inputs, bool inplace, bool clean, bool softlist, string outDir,
+ DatFile inputDat, out DatFile userData, string gamename, string romname, string romtype, long sgt, long slt, long seq, string crc,
+ string md5, string sha1, ItemStatus itemStatus, bool trim, bool single, string root, int maxDegreeOfParallelism, Logger logger)
+ {
+ DatFile[] datHeaders = new DatFile[inputs.Count];
+ DateTime start = DateTime.Now;
+ logger.User("Processing individual DATs");
+
+ Parallel.For(0,
+ inputs.Count,
+ new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
+ i =>
+ {
+ string input = inputs[i];
+ logger.User("Adding DAT: " + input.Split('¬')[0]);
+ datHeaders[i] = new DatFile
+ {
+ OutputFormat = (inputDat.OutputFormat != 0 ? inputDat.OutputFormat : 0),
+ Files = new Dictionary>(),
+ MergeRoms = inputDat.MergeRoms,
+ };
+
+ datHeaders[i].Parse(input.Split('¬')[0], i, 0, gamename, romname, romtype, sgt, slt, seq,
+ crc, md5, sha1, itemStatus, trim, single, root, logger, true, clean, softlist);
+ });
+
+ logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ logger.User("Populating internal DAT");
+ userData = (DatFile)inputDat.CloneHeader();
+ userData.Files = new Dictionary>();
+ for (int i = 0; i < inputs.Count; i++)
+ {
+ List keys = datHeaders[i].Files.Keys.ToList();
+ foreach (string key in keys)
+ {
+ if (userData.Files.ContainsKey(key))
+ {
+ userData.Files[key].AddRange(datHeaders[i].Files[key]);
+ }
+ else
+ {
+ userData.Files.Add(key, datHeaders[i].Files[key]);
+ }
+ datHeaders[i].Files.Remove(key);
+ }
+ datHeaders[i].Files = null;
+ }
+
+ logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
+
+ return datHeaders.ToList();
+ }
+
+ #endregion
+
#region DAT Splitting
///
@@ -4552,7 +4427,7 @@ Please check the log folder if the stats scrolled offscreen");
string newExtBString = string.Join(",", newExtB);
// Get the file format
- OutputFormat outputFormat = GetOutputFormat(filename, logger);
+ OutputFormat outputFormat = FileTools.GetOutputFormat(filename, logger);
if (outputFormat == 0)
{
return true;
@@ -4690,7 +4565,7 @@ Please check the log folder if the stats scrolled offscreen");
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
// Get the file format
- OutputFormat outputFormat = GetOutputFormat(filename, logger);
+ OutputFormat outputFormat = FileTools.GetOutputFormat(filename, logger);
if (outputFormat == 0)
{
return true;
@@ -4950,7 +4825,7 @@ Please check the log folder if the stats scrolled offscreen");
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
// Get the file format
- OutputFormat outputFormat = GetOutputFormat(filename, logger);
+ OutputFormat outputFormat = FileTools.GetOutputFormat(filename, logger);
if (outputFormat == 0)
{
return true;
@@ -5112,5 +4987,97 @@ Please check the log folder if the stats scrolled offscreen");
}
#endregion
+
+ #region Statistics
+
+ ///
+ /// Output the stats for a list of input dats as files in a human-readable format
+ ///
+ /// List of input files and folders
+ /// True if single DAT stats are output, false otherwise
+ /// Logger object for file and console output
+ public static void OutputStats(List inputs, bool single, Logger logger)
+ {
+ // Make sure we have all files
+ List newinputs = new List();
+ foreach (string input in inputs)
+ {
+ if (File.Exists(input))
+ {
+ newinputs.Add(input);
+ }
+ if (Directory.Exists(input))
+ {
+ foreach (string file in Directory.GetFiles(input, "*", SearchOption.AllDirectories))
+ {
+ newinputs.Add(file);
+ }
+ }
+ }
+
+ // Init all total variables
+ long totalSize = 0;
+ long totalGame = 0;
+ long totalRom = 0;
+ long totalDisk = 0;
+ long totalCRC = 0;
+ long totalMD5 = 0;
+ long totalSHA1 = 0;
+ long totalNodump = 0;
+
+ /// Now process each of the input files
+ foreach (string filename in newinputs)
+ {
+ logger.Log("Beginning stat collection for '" + filename + "'");
+ List games = new List();
+ DatFile datdata = new DatFile();
+ datdata.Parse(filename, 0, 0, logger);
+ SortedDictionary> newroms = BucketByGame(datdata.Files, false, true, logger, false);
+
+ // Output single DAT stats (if asked)
+ if (single)
+ {
+ logger.User(@"\nFor file '" + filename + @"':
+--------------------------------------------------");
+ datdata.OutputStats(logger);
+ }
+ else
+ {
+ logger.User("Adding stats for file '" + filename + "'\n");
+ }
+
+ // Add single DAT stats to totals
+ totalSize += datdata.TotalSize;
+ totalGame += newroms.Count;
+ totalRom += datdata.RomCount;
+ totalDisk += datdata.DiskCount;
+ totalCRC += datdata.CRCCount;
+ totalMD5 += datdata.MD5Count;
+ totalSHA1 += datdata.SHA1Count;
+ totalNodump += datdata.NodumpCount;
+ }
+
+ // Output total DAT stats
+ if (!single) { logger.User(""); }
+ DatFile totaldata = new DatFile
+ {
+ TotalSize = totalSize,
+ RomCount = totalRom,
+ DiskCount = totalDisk,
+ CRCCount = totalCRC,
+ MD5Count = totalMD5,
+ SHA1Count = totalSHA1,
+ NodumpCount = totalNodump,
+ };
+ logger.User(@"For ALL DATs found
+--------------------------------------------------");
+ totaldata.OutputStats(logger, game: totalGame);
+ logger.User(@"
+Please check the log folder if the stats scrolled offscreen");
+ }
+
+ #endregion
+
+ #endregion // Static Methods
}
}
diff --git a/SabreTools.Helper/Tools/FileTools.cs b/SabreTools.Helper/Tools/FileTools.cs
index 3005c657..86da0a87 100644
--- a/SabreTools.Helper/Tools/FileTools.cs
+++ b/SabreTools.Helper/Tools/FileTools.cs
@@ -670,6 +670,55 @@ namespace SabreTools.Helper
#region File Information
+ ///
+ /// Get what type of DAT the input file is
+ ///
+ /// Name of the file to be parsed
+ /// The OutputFormat corresponding to the DAT
+ /// There is currently no differentiation between XML and SabreDAT here
+ public static OutputFormat GetOutputFormat(string filename, Logger logger)
+ {
+ // Limit the output formats based on extension
+ string ext = Path.GetExtension(filename).ToLowerInvariant();
+ if (ext != ".dat" && ext != ".xml")
+ {
+ return 0;
+ }
+
+ // Read the input file, if possible
+ logger.Log("Attempting to read file: \"" + filename + "\"");
+
+ // Check if file exists
+ if (!File.Exists(filename))
+ {
+ logger.Warning("File '" + filename + "' could not read from!");
+ return 0;
+ }
+
+ try
+ {
+ StreamReader sr = File.OpenText(filename);
+ string first = sr.ReadLine();
+ sr.Dispose();
+ if (first.Contains("<") && first.Contains(">"))
+ {
+ return OutputFormat.Xml;
+ }
+ else if (first.Contains("[") && first.Contains("]"))
+ {
+ return OutputFormat.RomCenter;
+ }
+ else
+ {
+ return OutputFormat.ClrMamePro;
+ }
+ }
+ catch (Exception)
+ {
+ return 0;
+ }
+ }
+
///
/// Retrieve file information for a single file
///