diff --git a/SabreTools.Helper/Objects/SimpleSort.cs b/SabreTools.Helper/Objects/SimpleSort.cs
index 7f67d44d..43099594 100644
--- a/SabreTools.Helper/Objects/SimpleSort.cs
+++ b/SabreTools.Helper/Objects/SimpleSort.cs
@@ -280,249 +280,6 @@ namespace SabreTools.Helper
return success;
}
- ///
- /// Process the DAT and find all matches in input files and folders
- ///
- /// True if rebuilding was a success, false otherwise
- ///
- /// This implemenation of the code should do the following:
- /// 1) Get all file names from the input files/folders (parallel)
- /// 2) Loop through and get the file info from every file (including headerless)
- /// 3) Find all duplicate files in the input DAT(s)
- /// 4) Order by output game
- /// 5) Rebuild all files
- ///
- public bool RebuiltToOutputAlternate()
- {
- bool success = true;
-
- #region Find all files
-
- // Create a list of just files from inputs
- _logger.User("Finding all files...");
- List files = new List();
- Parallel.ForEach(_inputs,
- new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
- input =>
- {
- if (File.Exists(input))
- {
- _logger.Verbose("File found: '" + input + "'");
- lock (files)
- {
- files.Add(Path.GetFullPath(input));
- }
- }
- else if (Directory.Exists(input))
- {
- _logger.Verbose("Directory found: '" + input + "'");
-
- List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
- Parallel.ForEach(infiles,
- new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
- file =>
- {
- _logger.Verbose("File found: '" + input + "'");
- lock (files)
- {
- files.Add(Path.GetFullPath(file));
- }
- });
- }
- else
- {
- _logger.Error("'" + input + "' is not a file or directory!");
- }
- });
- _logger.User("Finding files complete!");
-
- #endregion
-
- #region Get source file information
-
- // Now loop through all of the files and check them, DFD style
- _logger.User("Getting source file information...");
- DatFile matchdat = new DatFile
- {
- Files = new SortedDictionary>(),
- };
- foreach (string file in files)
- {
- // Get if the file should be scanned internally and externally
- bool shouldExternalScan, shouldInternalScan;
- ArchiveTools.GetInternalExternalProcess(file, _7z, _gz, _rar, _zip, _logger, out shouldExternalScan, out shouldInternalScan);
-
- // Hash and match the external files
- if (shouldExternalScan)
- {
- RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
- }
-
- // If we should scan the file as an archive
- if (shouldInternalScan)
- {
- // If external scanning is enabled, use that method instead
- if (_quickScan)
- {
- _logger.Verbose("Beginning quick scan of contents from '" + file + "'");
- List internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger);
- _logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
-
- // Now add all of the roms to the DAT
- for (int i = 0; i < internalRomData.Count; i++)
- {
- RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
- }
- }
- // Otherwise, try to extract the file to the temp folder
- else
- {
- // Now, if the file is a supported archive type, also run on all files within
- bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _7z, _gz, _rar, _zip, _logger);
-
- // If we succeeded in extracting, loop through the files
- if (!encounteredErrors)
- {
- List extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
- foreach (string extractedFile in extractedFiles)
- {
- RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
- }
- }
- // Otherwise, skip extracting and just get information on the file itself (if we didn't already)
- else if (!shouldExternalScan)
- {
- RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
- }
-
- // Clean the temp directory for the next round
- if (Directory.Exists(_tempDir))
- {
- FileTools.CleanDirectory(_tempDir);
- }
- }
- }
- }
- _logger.User("Getting source file information complete!");
-
- #endregion
-
- #region Find all files to rebuild and bucket by game
-
- // Create a dictionary of from/to Rom mappings
- Dictionary toFromMap = new Dictionary();
-
- // Now populate it
- foreach (string key in matchdat.Files.Keys)
- {
- foreach (DatItem rom in matchdat.Files[key])
- {
- List matched = rom.GetDuplicates(_datdata, _logger, true);
- foreach (DatItem match in matched)
- {
- try
- {
- toFromMap.Add(match, rom);
- }
- catch { }
- }
- }
- }
-
- // Then bucket the keys by game for better output
- SortedDictionary> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger);
-
- #endregion
-
- #region Rebuild all files
-
- // At this point, we have "toFromMap" which maps output files to input files as well as
- // as SortedDictionary called keysByGame which is the output files sorted by game in
- // alphabetical order. We should be able to use these to do everything we need =)
-
- // Now write out each game sequentially
- foreach (string key in keysByGame.Keys)
- {
-
- }
-
- #endregion
-
- return success;
- }
-
- ///
- /// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
- ///
- /// Name of the file to attempt to add
- /// Reference to the Dat to add to
- /// Logger object for file and console output
- /// True if the file could be added, false otherwise
- public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
- {
- Rom rom = FileTools.GetFileInfo(file, logger);
-
- // If we have a blank RomData, it's an error
- if (rom.Name == null)
- {
- return false;
- }
-
- // Otherwise, set the machine name as the full path to the file
- rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
-
- // Add the rom information to the Dat
- string key = rom.Size + "-" + rom.CRC;
- if (matchdat.Files.ContainsKey(key))
- {
- matchdat.Files[key].Add(rom);
- }
- else
- {
- List temp = new List();
- temp.Add(rom);
- matchdat.Files.Add(key, temp);
- }
-
- // Now attempt to see if the file has a header
- FileStream input = File.OpenRead(file);
- SkipperRule rule = Skipper.GetMatchingRule(input, "", _logger);
-
- // If there's a match, get the new information from the stream
- if (rule.Tests != null && rule.Tests.Count != 0)
- {
- // Create the input and output streams
- MemoryStream output = new MemoryStream();
-
- // Transform the stream and get the information from it
- rule.TransformStream(input, output, _logger, false, true);
- Rom romNH = FileTools.GetStreamInfo(output, output.Length);
- romNH.Name = "HEAD::" + rom.Name;
- romNH.MachineName = rom.MachineName;
-
- // Add the rom information to the Dat
- key = romNH.Size + "-" + romNH.CRC;
- if (matchdat.Files.ContainsKey(key))
- {
- matchdat.Files[key].Add(romNH);
- }
- else
- {
- List temp = new List();
- temp.Add(romNH);
- matchdat.Files.Add(key, temp);
- }
-
- // Dispose of the stream
- output.Dispose();
- }
-
- // Dispose of the stream
- input.Dispose();
-
- return true;
- }
-
///
/// Process an individual file against the DAT for rebuilding
///
@@ -871,6 +628,249 @@ namespace SabreTools.Helper
return success;
}
+ ///
+ /// Process the DAT and find all matches in input files and folders
+ ///
+ /// True if rebuilding was a success, false otherwise
+ ///
+ /// This implemenation of the code should do the following:
+ /// 1) Get all file names from the input files/folders (parallel)
+ /// 2) Loop through and get the file info from every file (including headerless)
+ /// 3) Find all duplicate files in the input DAT(s)
+ /// 4) Order by output game
+ /// 5) Rebuild all files
+ ///
+ public bool RebuiltToOutputAlternate()
+ {
+ bool success = true;
+
+ #region Find all files
+
+ // Create a list of just files from inputs
+ _logger.User("Finding all files...");
+ List files = new List();
+ Parallel.ForEach(_inputs,
+ new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
+ input =>
+ {
+ if (File.Exists(input))
+ {
+ _logger.Verbose("File found: '" + input + "'");
+ lock (files)
+ {
+ files.Add(Path.GetFullPath(input));
+ }
+ }
+ else if (Directory.Exists(input))
+ {
+ _logger.Verbose("Directory found: '" + input + "'");
+
+ List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
+ Parallel.ForEach(infiles,
+ new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
+ file =>
+ {
+ _logger.Verbose("File found: '" + input + "'");
+ lock (files)
+ {
+ files.Add(Path.GetFullPath(file));
+ }
+ });
+ }
+ else
+ {
+ _logger.Error("'" + input + "' is not a file or directory!");
+ }
+ });
+ _logger.User("Finding files complete!");
+
+ #endregion
+
+ #region Get source file information
+
+ // Now loop through all of the files and check them, DFD style
+ _logger.User("Getting source file information...");
+ DatFile matchdat = new DatFile
+ {
+ Files = new SortedDictionary>(),
+ };
+ foreach (string file in files)
+ {
+ // Get if the file should be scanned internally and externally
+ bool shouldExternalScan, shouldInternalScan;
+ ArchiveTools.GetInternalExternalProcess(file, _7z, _gz, _rar, _zip, _logger, out shouldExternalScan, out shouldInternalScan);
+
+ // Hash and match the external files
+ if (shouldExternalScan)
+ {
+ RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
+ }
+
+ // If we should scan the file as an archive
+ if (shouldInternalScan)
+ {
+ // If external scanning is enabled, use that method instead
+ if (_quickScan)
+ {
+ _logger.Verbose("Beginning quick scan of contents from '" + file + "'");
+ List internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger);
+ _logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
+
+ // Now add all of the roms to the DAT
+ for (int i = 0; i < internalRomData.Count; i++)
+ {
+ RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
+ }
+ }
+ // Otherwise, try to extract the file to the temp folder
+ else
+ {
+ // Now, if the file is a supported archive type, also run on all files within
+ bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _7z, _gz, _rar, _zip, _logger);
+
+ // If we succeeded in extracting, loop through the files
+ if (!encounteredErrors)
+ {
+ List extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
+ foreach (string extractedFile in extractedFiles)
+ {
+ RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
+ }
+ }
+ // Otherwise, skip extracting and just get information on the file itself (if we didn't already)
+ else if (!shouldExternalScan)
+ {
+ RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
+ }
+
+ // Clean the temp directory for the next round
+ if (Directory.Exists(_tempDir))
+ {
+ FileTools.CleanDirectory(_tempDir);
+ }
+ }
+ }
+ }
+ _logger.User("Getting source file information complete!");
+
+ #endregion
+
+ #region Find all files to rebuild and bucket by game
+
+ // Create a dictionary of from/to Rom mappings
+ Dictionary toFromMap = new Dictionary();
+
+ // Now populate it
+ foreach (string key in matchdat.Files.Keys)
+ {
+ foreach (DatItem rom in matchdat.Files[key])
+ {
+ List matched = rom.GetDuplicates(_datdata, _logger, true);
+ foreach (DatItem match in matched)
+ {
+ try
+ {
+ toFromMap.Add(match, rom);
+ }
+ catch { }
+ }
+ }
+ }
+
+ // Then bucket the keys by game for better output
+ SortedDictionary> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger);
+
+ #endregion
+
+ #region Rebuild all files
+
+ // At this point, we have "toFromMap" which maps output files to input files as well as
+ // as SortedDictionary called keysByGame which is the output files sorted by game in
+ // alphabetical order. We should be able to use these to do everything we need =)
+
+ // Now write out each game sequentially
+ foreach (string key in keysByGame.Keys)
+ {
+
+ }
+
+ #endregion
+
+ return success;
+ }
+
+ ///
+ /// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
+ ///
+ /// Name of the file to attempt to add
+ /// Reference to the Dat to add to
+ /// Logger object for file and console output
+ /// True if the file could be added, false otherwise
+ public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
+ {
+ Rom rom = FileTools.GetFileInfo(file, logger);
+
+ // If we have a blank RomData, it's an error
+ if (rom.Name == null)
+ {
+ return false;
+ }
+
+ // Otherwise, set the machine name as the full path to the file
+ rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
+
+ // Add the rom information to the Dat
+ string key = rom.Size + "-" + rom.CRC;
+ if (matchdat.Files.ContainsKey(key))
+ {
+ matchdat.Files[key].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ matchdat.Files.Add(key, temp);
+ }
+
+ // Now attempt to see if the file has a header
+ FileStream input = File.OpenRead(file);
+ SkipperRule rule = Skipper.GetMatchingRule(input, "", _logger);
+
+ // If there's a match, get the new information from the stream
+ if (rule.Tests != null && rule.Tests.Count != 0)
+ {
+ // Create the input and output streams
+ MemoryStream output = new MemoryStream();
+
+ // Transform the stream and get the information from it
+ rule.TransformStream(input, output, _logger, false, true);
+ Rom romNH = FileTools.GetStreamInfo(output, output.Length);
+ romNH.Name = "HEAD::" + rom.Name;
+ romNH.MachineName = rom.MachineName;
+
+ // Add the rom information to the Dat
+ key = romNH.Size + "-" + romNH.CRC;
+ if (matchdat.Files.ContainsKey(key))
+ {
+ matchdat.Files[key].Add(romNH);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(romNH);
+ matchdat.Files.Add(key, temp);
+ }
+
+ // Dispose of the stream
+ output.Dispose();
+ }
+
+ // Dispose of the stream
+ input.Dispose();
+
+ return true;
+ }
+
///
/// Clean an individual folder based on the DAT
///
@@ -1040,6 +1040,17 @@ namespace SabreTools.Helper
// Do an external scan of the file, if necessary
if (shouldExternalProcess)
{
+ // If a DAT is defined, we want to make sure that this file is not in there
+ if (_datdata != null && _datdata.Files.Count > 0)
+ {
+ Rom rom = FileTools.GetFileInfo(input, _logger);
+ if (rom.GetDuplicates(_datdata, _logger).Count > 0)
+ {
+ _logger.User("File '" + input + "' existed in the DAT, skipping...");
+ continue;
+ }
+ }
+
_logger.User("Processing file " + input);
success &= ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger);
}
@@ -1056,6 +1067,17 @@ namespace SabreTools.Helper
_logger.Verbose("Archive found! Successfully extracted");
foreach (string file in Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories))
{
+ // If a DAT is defined, we want to make sure that this file is not in there
+ if (_datdata != null && _datdata.Files.Count > 0)
+ {
+ Rom rom = FileTools.GetFileInfo(file, _logger);
+ if (rom.GetDuplicates(_datdata, _logger).Count > 0)
+ {
+ _logger.User("File '" + file + "' existed in the DAT, skipping...");
+ continue;
+ }
+ }
+
_logger.User("Processing extracted file " + file);
success &= ArchiveTools.WriteTorrentGZ(file, _outDir, _romba, _logger);
}
diff --git a/SabreTools.Helper/README.1ST b/SabreTools.Helper/README.1ST
index c66369cb..6c7874d0 100644
--- a/SabreTools.Helper/README.1ST
+++ b/SabreTools.Helper/README.1ST
@@ -715,7 +715,8 @@ Options:
-c, --convert Enable conversion of input files to TGZ
This allows conversion of a folder or set of folders to TorrentGZ format without
requiring a DAT to rebuild from. It is only useful in a small amount of situations at
- the present, but it is mostly meant for Romba compatibility at the present.
+ the present, but it is mostly meant for Romba compatibility at the present. If a DAT
+ is supplied, then files NOT matching the DAT will be written out only.
-tzip Enable TorrentZip output
Instead of outputting the files to standard ZIP archives, files will be rebuilt to