claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[SimpleSort] Use convert as a filter

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-05 16:54:52 -07:00
parent 78582b7377
commit 1fa4cfa65f
2 changed files with 267 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

508
SabreTools.Helper/Objects/SimpleSort.cs
View File

@@ -280,249 +280,6 @@ namespace SabreTools.Helper
return success; return success;
} }
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <returns>True if rebuilding was a success, false otherwise</returns>
/// <remarks>
/// This implemenation of the code should do the following:
/// 1) Get all file names from the input files/folders (parallel)
/// 2) Loop through and get the file info from every file (including headerless)
/// 3) Find all duplicate files in the input DAT(s)
/// 4) Order by output game
/// 5) Rebuild all files
/// </remarks>
public bool RebuiltToOutputAlternate()
{
bool success = true;
#region Find all files
// Create a list of just files from inputs
_logger.User("Finding all files...");
List<string> files = new List<string>();
Parallel.ForEach(_inputs,
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
input =>
{
if (File.Exists(input))
{
_logger.Verbose("File found: '" + input + "'");
lock (files)
{
files.Add(Path.GetFullPath(input));
}
}
else if (Directory.Exists(input))
{
_logger.Verbose("Directory found: '" + input + "'");
List<string> infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(infiles,
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
file =>
{
_logger.Verbose("File found: '" + input + "'");
lock (files)
{
files.Add(Path.GetFullPath(file));
}
});
}
else
{
_logger.Error("'" + input + "' is not a file or directory!");
}
});
_logger.User("Finding files complete!");
#endregion
#region Get source file information
// Now loop through all of the files and check them, DFD style
_logger.User("Getting source file information...");
DatFile matchdat = new DatFile
{
Files = new SortedDictionary<string, List<DatItem>>(),
};
foreach (string file in files)
{
// Get if the file should be scanned internally and externally
bool shouldExternalScan, shouldInternalScan;
ArchiveTools.GetInternalExternalProcess(file, _7z, _gz, _rar, _zip, _logger, out shouldExternalScan, out shouldInternalScan);
// Hash and match the external files
if (shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// If we should scan the file as an archive
if (shouldInternalScan)
{
// If external scanning is enabled, use that method instead
if (_quickScan)
{
_logger.Verbose("Beginning quick scan of contents from '" + file + "'");
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger);
_logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
// Now add all of the roms to the DAT
for (int i = 0; i < internalRomData.Count; i++)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
}
// Otherwise, try to extract the file to the temp folder
else
{
// Now, if the file is a supported archive type, also run on all files within
bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _7z, _gz, _rar, _zip, _logger);
// If we succeeded in extracting, loop through the files
if (!encounteredErrors)
{
List<string> extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
foreach (string extractedFile in extractedFiles)
{
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
}
}
// Otherwise, skip extracting and just get information on the file itself (if we didn't already)
else if (!shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// Clean the temp directory for the next round
if (Directory.Exists(_tempDir))
{
FileTools.CleanDirectory(_tempDir);
}
}
}
}
_logger.User("Getting source file information complete!");
#endregion
#region Find all files to rebuild and bucket by game
// Create a dictionary of from/to Rom mappings
Dictionary<DatItem, DatItem> toFromMap = new Dictionary<DatItem, DatItem>();
// Now populate it
foreach (string key in matchdat.Files.Keys)
{
foreach (DatItem rom in matchdat.Files[key])
{
List<DatItem> matched = rom.GetDuplicates(_datdata, _logger, true);
foreach (DatItem match in matched)
{
try
{
toFromMap.Add(match, rom);
}
catch { }
}
}
}
// Then bucket the keys by game for better output
SortedDictionary<string, List<DatItem>> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger);
#endregion
#region Rebuild all files
// At this point, we have "toFromMap" which maps output files to input files as well as
// as SortedDictionary called keysByGame which is the output files sorted by game in
// alphabetical order. We should be able to use these to do everything we need =)
// Now write out each game sequentially
foreach (string key in keysByGame.Keys)
{
}
#endregion
return success;
}
/// <summary>
/// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
/// </summary>
/// <param name="file">Name of the file to attempt to add</param>
/// <param name="matchdat">Reference to the Dat to add to</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if the file could be added, false otherwise</returns>
public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
{
Rom rom = FileTools.GetFileInfo(file, logger);
// If we have a blank RomData, it's an error
if (rom.Name == null)
{
return false;
}
// Otherwise, set the machine name as the full path to the file
rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
// Add the rom information to the Dat
string key = rom.Size + "-" + rom.CRC;
if (matchdat.Files.ContainsKey(key))
{
matchdat.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
matchdat.Files.Add(key, temp);
}
// Now attempt to see if the file has a header
FileStream input = File.OpenRead(file);
SkipperRule rule = Skipper.GetMatchingRule(input, "", _logger);
// If there's a match, get the new information from the stream
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the input and output streams
MemoryStream output = new MemoryStream();
// Transform the stream and get the information from it
rule.TransformStream(input, output, _logger, false, true);
Rom romNH = FileTools.GetStreamInfo(output, output.Length);
romNH.Name = "HEAD::" + rom.Name;
romNH.MachineName = rom.MachineName;
// Add the rom information to the Dat
key = romNH.Size + "-" + romNH.CRC;
if (matchdat.Files.ContainsKey(key))
{
matchdat.Files[key].Add(romNH);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(romNH);
matchdat.Files.Add(key, temp);
}
// Dispose of the stream
output.Dispose();
}
// Dispose of the stream
input.Dispose();
return true;
}
/// <summary> /// <summary>
/// Process an individual file against the DAT for rebuilding /// Process an individual file against the DAT for rebuilding
/// </summary> /// </summary>
@@ -871,6 +628,249 @@ namespace SabreTools.Helper
return success; return success;
} }
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <returns>True if rebuilding was a success, false otherwise</returns>
/// <remarks>
/// This implemenation of the code should do the following:
/// 1) Get all file names from the input files/folders (parallel)
/// 2) Loop through and get the file info from every file (including headerless)
/// 3) Find all duplicate files in the input DAT(s)
/// 4) Order by output game
/// 5) Rebuild all files
/// </remarks>
public bool RebuiltToOutputAlternate()
{
bool success = true;
#region Find all files
// Create a list of just files from inputs
_logger.User("Finding all files...");
List<string> files = new List<string>();
Parallel.ForEach(_inputs,
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
input =>
{
if (File.Exists(input))
{
_logger.Verbose("File found: '" + input + "'");
lock (files)
{
files.Add(Path.GetFullPath(input));
}
}
else if (Directory.Exists(input))
{
_logger.Verbose("Directory found: '" + input + "'");
List<string> infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(infiles,
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
file =>
{
_logger.Verbose("File found: '" + input + "'");
lock (files)
{
files.Add(Path.GetFullPath(file));
}
});
}
else
{
_logger.Error("'" + input + "' is not a file or directory!");
}
});
_logger.User("Finding files complete!");
#endregion
#region Get source file information
// Now loop through all of the files and check them, DFD style
_logger.User("Getting source file information...");
DatFile matchdat = new DatFile
{
Files = new SortedDictionary<string, List<DatItem>>(),
};
foreach (string file in files)
{
// Get if the file should be scanned internally and externally
bool shouldExternalScan, shouldInternalScan;
ArchiveTools.GetInternalExternalProcess(file, _7z, _gz, _rar, _zip, _logger, out shouldExternalScan, out shouldInternalScan);
// Hash and match the external files
if (shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// If we should scan the file as an archive
if (shouldInternalScan)
{
// If external scanning is enabled, use that method instead
if (_quickScan)
{
_logger.Verbose("Beginning quick scan of contents from '" + file + "'");
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger);
_logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
// Now add all of the roms to the DAT
for (int i = 0; i < internalRomData.Count; i++)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
}
// Otherwise, try to extract the file to the temp folder
else
{
// Now, if the file is a supported archive type, also run on all files within
bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _7z, _gz, _rar, _zip, _logger);
// If we succeeded in extracting, loop through the files
if (!encounteredErrors)
{
List<string> extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
foreach (string extractedFile in extractedFiles)
{
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
}
}
// Otherwise, skip extracting and just get information on the file itself (if we didn't already)
else if (!shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// Clean the temp directory for the next round
if (Directory.Exists(_tempDir))
{
FileTools.CleanDirectory(_tempDir);
}
}
}
}
_logger.User("Getting source file information complete!");
#endregion
#region Find all files to rebuild and bucket by game
// Create a dictionary of from/to Rom mappings
Dictionary<DatItem, DatItem> toFromMap = new Dictionary<DatItem, DatItem>();
// Now populate it
foreach (string key in matchdat.Files.Keys)
{
foreach (DatItem rom in matchdat.Files[key])
{
List<DatItem> matched = rom.GetDuplicates(_datdata, _logger, true);
foreach (DatItem match in matched)
{
try
{
toFromMap.Add(match, rom);
}
catch { }
}
}
}
// Then bucket the keys by game for better output
SortedDictionary<string, List<DatItem>> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger);
#endregion
#region Rebuild all files
// At this point, we have "toFromMap" which maps output files to input files as well as
// as SortedDictionary called keysByGame which is the output files sorted by game in
// alphabetical order. We should be able to use these to do everything we need =)
// Now write out each game sequentially
foreach (string key in keysByGame.Keys)
{
}
#endregion
return success;
}
/// <summary>
/// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
/// </summary>
/// <param name="file">Name of the file to attempt to add</param>
/// <param name="matchdat">Reference to the Dat to add to</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if the file could be added, false otherwise</returns>
public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
{
Rom rom = FileTools.GetFileInfo(file, logger);
// If we have a blank RomData, it's an error
if (rom.Name == null)
{
return false;
}
// Otherwise, set the machine name as the full path to the file
rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
// Add the rom information to the Dat
string key = rom.Size + "-" + rom.CRC;
if (matchdat.Files.ContainsKey(key))
{
matchdat.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
matchdat.Files.Add(key, temp);
}
// Now attempt to see if the file has a header
FileStream input = File.OpenRead(file);
SkipperRule rule = Skipper.GetMatchingRule(input, "", _logger);
// If there's a match, get the new information from the stream
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the input and output streams
MemoryStream output = new MemoryStream();
// Transform the stream and get the information from it
rule.TransformStream(input, output, _logger, false, true);
Rom romNH = FileTools.GetStreamInfo(output, output.Length);
romNH.Name = "HEAD::" + rom.Name;
romNH.MachineName = rom.MachineName;
// Add the rom information to the Dat
key = romNH.Size + "-" + romNH.CRC;
if (matchdat.Files.ContainsKey(key))
{
matchdat.Files[key].Add(romNH);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(romNH);
matchdat.Files.Add(key, temp);
}
// Dispose of the stream
output.Dispose();
}
// Dispose of the stream
input.Dispose();
return true;
}
/// <summary> /// <summary>
/// Clean an individual folder based on the DAT /// Clean an individual folder based on the DAT
/// </summary> /// </summary>
@@ -1040,6 +1040,17 @@ namespace SabreTools.Helper
// Do an external scan of the file, if necessary // Do an external scan of the file, if necessary
if (shouldExternalProcess) if (shouldExternalProcess)
{ {
// If a DAT is defined, we want to make sure that this file is not in there
if (_datdata != null && _datdata.Files.Count > 0)
{
Rom rom = FileTools.GetFileInfo(input, _logger);
if (rom.GetDuplicates(_datdata, _logger).Count > 0)
{
_logger.User("File '" + input + "' existed in the DAT, skipping...");
continue;
}
}
_logger.User("Processing file " + input); _logger.User("Processing file " + input);
success &= ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger); success &= ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger);
} }
@@ -1056,6 +1067,17 @@ namespace SabreTools.Helper
_logger.Verbose("Archive found! Successfully extracted"); _logger.Verbose("Archive found! Successfully extracted");
foreach (string file in Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories)) foreach (string file in Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories))
{ {
// If a DAT is defined, we want to make sure that this file is not in there
if (_datdata != null && _datdata.Files.Count > 0)
{
Rom rom = FileTools.GetFileInfo(file, _logger);
if (rom.GetDuplicates(_datdata, _logger).Count > 0)
{
_logger.User("File '" + file + "' existed in the DAT, skipping...");
continue;
}
}
_logger.User("Processing extracted file " + file); _logger.User("Processing extracted file " + file);
success &= ArchiveTools.WriteTorrentGZ(file, _outDir, _romba, _logger); success &= ArchiveTools.WriteTorrentGZ(file, _outDir, _romba, _logger);
} }

3
SabreTools.Helper/README.1ST
View File

@@ -715,7 +715,8 @@ Options:
-c, --convert Enable conversion of input files to TGZ -c, --convert Enable conversion of input files to TGZ
This allows conversion of a folder or set of folders to TorrentGZ format without This allows conversion of a folder or set of folders to TorrentGZ format without
requiring a DAT to rebuild from. It is only useful in a small amount of situations at requiring a DAT to rebuild from. It is only useful in a small amount of situations at
the present, but it is mostly meant for Romba compatibility at the present. the present, but it is mostly meant for Romba compatibility at the present. If a DAT
is supplied, then files NOT matching the DAT will be written out only.
-tzip Enable TorrentZip output -tzip Enable TorrentZip output
Instead of outputting the files to standard ZIP archives, files will be rebuilt to Instead of outputting the files to standard ZIP archives, files will be rebuilt to