claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DATFromDirParallel] Fixes to make DFDP viable

Browse Source
This commit is contained in:
Matt Nadareski
2016-09-06 14:58:00 -07:00
parent 06980d88e4
commit bb70e1bf43
1 changed files with 33 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

275
SabreTools.Helper/Objects/DATFromDirParallel.cs
View File

@@ -19,7 +19,6 @@ namespace SabreTools
private string _tempDir; private string _tempDir;
// User specified inputs // User specified inputs
private List<String> _inputs;
private Dat _datdata; private Dat _datdata;
private bool _noMD5; private bool _noMD5;
private bool _noSHA1; private bool _noSHA1;
@@ -39,7 +38,7 @@ namespace SabreTools
/// <summary> /// <summary>
/// Create a new DATFromDir object /// Create a new DATFromDir object
/// </summary> /// </summary>
/// <param name="inputs">A List of Strings representing the files and folders to be DATted</param> /// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="datdata">DatData object representing the requested output DAT</param> /// <param name="datdata">DatData object representing the requested output DAT</param>
/// <param name="noMD5">True if MD5 hashes should be skipped over, false otherwise</param> /// <param name="noMD5">True if MD5 hashes should be skipped over, false otherwise</param>
/// <param name="noSHA1">True if SHA-1 hashes should be skipped over, false otherwise</param> /// <param name="noSHA1">True if SHA-1 hashes should be skipped over, false otherwise</param>
@@ -49,10 +48,11 @@ namespace SabreTools
/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param> /// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
/// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param> /// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param>
/// <param name="logger">Logger object for console and file output</param> /// <param name="logger">Logger object for console and file output</param>
public DATFromDirParallel(List<String> inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger) public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger)
{ {
_inputs = inputs; _basePath = basePath;
_datdata = datdata; _datdata = datdata;
_datdata.Files.Add("null", new List<Rom>(10));
_noMD5 = noMD5; _noMD5 = noMD5;
_noSHA1 = noSHA1; _noSHA1 = noSHA1;
_bare = bare; _bare = bare;
@@ -69,10 +69,6 @@ namespace SabreTools
/// <remarks>Try to get the hashing multithreaded (either on a per-hash or per-file level)</remarks> /// <remarks>Try to get the hashing multithreaded (either on a per-hash or per-file level)</remarks>
public bool Start() public bool Start()
{ {
// Double check to see what it needs to be named
_basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : "");
_basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : "");
// If the description is defined but not the name, set the name from the description // If the description is defined but not the name, set the name from the description
if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description)) if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description))
{ {
@@ -88,198 +84,22 @@ namespace SabreTools
// If neither the name or description are defined, set them from the automatic values // If neither the name or description are defined, set them from the automatic values
else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description)) else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
{ {
if (_inputs.Count > 1) if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString()))
{ {
_datdata.Name = Environment.CurrentDirectory.Split(Path.DirectorySeparatorChar).Last(); _basePath = _basePath.Substring(0, _basePath.Length - 1);
} }
else _datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last();
{
if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
_basePath = _basePath.Substring(0, _basePath.Length - 1);
}
_datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last();
}
// If the name is still somehow empty, populate it with defaults
_datdata.Name = (String.IsNullOrEmpty(_datdata.Name) ? "Default" : _datdata.Name);
_datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")"); _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
} }
// Loop over each of the found paths, if any // Loop over the inputs
string lastparent = null; _logger.Log("Folder found: " + _basePath);
foreach (string path in _inputs)
// Process the files in all subfolders
Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories), item =>
{ {
// Set local paths and vars ProcessPossibleArchive(item);
_basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar); });
_basePath = Path.GetFullPath(_basePath);
// This is where the main loop would go
if (File.Exists(_basePath))
{
lastparent = ProcessPossibleArchive(_basePath, lastparent);
}
else if (Directory.Exists(_basePath))
{
_logger.Log("Folder found: " + _basePath);
// Process the files in the base folder first
Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.TopDirectoryOnly), item =>
{
lastparent = ProcessPossibleArchive(item, lastparent);
});
// Then process each of the subfolders themselves
string basePathBackup = _basePath;
foreach (string item in Directory.EnumerateDirectories(_basePath))
{
if (_datdata.Type != "SuperDAT")
{
_basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar);
_basePath = Path.GetFullPath(_basePath);
}
bool items = false;
Parallel.ForEach(Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories), subitem =>
{
items = true;
lastparent = ProcessPossibleArchive(subitem, lastparent);
});
// In romba mode we ignore empty folders completely
if (!_datdata.Romba)
{
// If there were no subitems, add a "blank" game to to the set (if not in Romba mode)
if (!items)
{
string actualroot = item.Remove(0, basePathBackup.Length);
Rom rom = new Rom
{
Name = "null",
Machine = new Machine
{
Name = (_datdata.Type == "SuperDAT" ?
(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
Path.DirectorySeparatorChar.ToString() :
"") + actualroot :
actualroot),
},
HashData = new Hash
{
Size = -1,
CRC = "null",
MD5 = "null",
SHA1 = "null",
},
};
string key = rom.HashData.Size + "-" + rom.HashData.CRC;
if (_datdata.Files.ContainsKey(key))
{
_datdata.Files[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
_datdata.Files.Add(key, temp);
}
}
// Now scour subdirectories for empties and add those as well (if not in Romba mode)
foreach (string subdir in Directory.EnumerateDirectories(item, "*", SearchOption.AllDirectories))
{
if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0)
{
string actualroot = subdir.Remove(0, basePathBackup.Length);
Rom rom = new Rom
{
Name = "null",
Machine = new Machine
{
Name = (_datdata.Type == "SuperDAT" ?
(actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
Path.DirectorySeparatorChar.ToString() :
"") + actualroot :
actualroot),
},
HashData = new Hash
{
Size = -1,
CRC = "null",
MD5 = "null",
SHA1 = "null",
},
};
string key = rom.HashData.Size + "-" + rom.HashData.CRC;
if (_datdata.Files.ContainsKey(key))
{
_datdata.Files[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
_datdata.Files.Add(key, temp);
}
}
}
}
}
_basePath = basePathBackup;
}
// If this somehow skips past the original sensors
else
{
_logger.Error(path + " is not a valid input!");
}
}
// Now output any empties to the stream (if not in Romba mode)
if (!_datdata.Romba)
{
List<string> keys = _datdata.Files.Keys.ToList();
foreach (string key in keys)
{
List<Rom> roms = _datdata.Files[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
// If we're in a mode that doesn't allow for actual empty folders, add the blank info
if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile)
{
rom.Type = ItemType.Rom;
rom.Name = "-";
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
}
string inkey = rom.HashData.Size + "-" + rom.HashData.CRC;
if (_datdata.Files.ContainsKey(inkey))
{
_datdata.Files[inkey].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
_datdata.Files.Add(inkey, temp);
}
lastparent = rom.Machine.Name;
}
}
// If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting
if (lastparent != null && _datdata.Files.Count == 0)
{
_datdata.Files.Add("temp", new List<Rom>());
}
}
return true; return true;
} }
@@ -288,14 +108,12 @@ namespace SabreTools
/// Check a given file for hashes, based on current settings /// Check a given file for hashes, based on current settings
/// </summary> /// </summary>
/// <param name="item">Filename of the item to be checked</param> /// <param name="item">Filename of the item to be checked</param>
/// <param name="lastparent">Name of the last parent rom to make sure that everything is grouped as well as possible</param>
/// <returns>New parent to be used</returns> /// <returns>New parent to be used</returns>
private string ProcessPossibleArchive(string item, string lastparent) private void ProcessPossibleArchive(string item)
{ {
// Define the temporary directory // Define the temporary directory
string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir); string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir);
tempdir += (tempdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? "" : Path.DirectorySeparatorChar.ToString()); tempdir = Path.Combine(tempdir, "__temp__", Path.GetFileNameWithoutExtension(item)) + Path.DirectorySeparatorChar;
tempdir += "__temp__" + Path.DirectorySeparatorChar;
// Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes) // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
if (_datdata.Romba) if (_datdata.Romba)
@@ -305,25 +123,15 @@ namespace SabreTools
// If the rom is valid, write it out // If the rom is valid, write it out
if (rom.Name != null) if (rom.Name != null)
{ {
string key = rom.HashData.Size + "-" + rom.HashData.CRC; _datdata.Files["null"].Add(rom);
if (_datdata.Files.ContainsKey(key))
{
_datdata.Files[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
_datdata.Files.Add(key, temp);
}
} }
else else
{ {
return string.Empty; return;
} }
_logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine); _logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
return rom.Machine.Name; return;
} }
// If both deep hash skip flags are set, do a quickscan // If both deep hash skip flags are set, do a quickscan
@@ -337,16 +145,14 @@ namespace SabreTools
List<Rom> extracted = FileTools.GetArchiveFileInfo(item, _logger); List<Rom> extracted = FileTools.GetArchiveFileInfo(item, _logger);
foreach (Rom rom in extracted) foreach (Rom rom in extracted)
{ {
lastparent = ProcessFileHelper(item, rom, _basePath, ProcessFileHelper(item, rom, _basePath,
Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) + (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length), _datdata);
Path.GetFileNameWithoutExtension(item)
), _datdata, lastparent);
} }
} }
// Otherwise, just get the info on the file itself // Otherwise, just get the info on the file itself
else if (!Directory.Exists(item) && File.Exists(item)) else if (!Directory.Exists(item) && File.Exists(item))
{ {
lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent); ProcessFile(item, _basePath, "", _datdata);
} }
} }
// Otherwise, attempt to extract the files to the temporary directory // Otherwise, attempt to extract the files to the temporary directory
@@ -367,13 +173,13 @@ namespace SabreTools
Parallel.ForEach(Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories), entry => Parallel.ForEach(Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories), entry =>
{ {
string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar); string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar);
lastparent = ProcessFile(Path.GetFullPath(entry), Path.GetFullPath(tempdir), ProcessFile(Path.GetFullPath(entry), Path.GetFullPath(tempdir),
(String.IsNullOrEmpty(tempbasepath) (String.IsNullOrEmpty(tempbasepath)
? "" ? ""
: (tempbasepath.Length < _basePath.Length : (tempbasepath.Length < _basePath.Length
? tempbasepath ? tempbasepath
: tempbasepath.Remove(0, _basePath.Length))) + : tempbasepath.Remove(0, _basePath.Length))) +
Path.GetFileNameWithoutExtension(item), _datdata, lastparent); Path.GetFileNameWithoutExtension(item), _datdata);
}); });
// Clear the temp directory // Clear the temp directory
@@ -385,11 +191,11 @@ namespace SabreTools
// Otherwise, just get the info on the file itself // Otherwise, just get the info on the file itself
else if (!Directory.Exists(item) && File.Exists(item)) else if (!Directory.Exists(item) && File.Exists(item))
{ {
lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent); ProcessFile(item, _basePath, Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) +
Path.GetFileNameWithoutExtension(item)
), _datdata);
} }
} }
return lastparent;
} }
/// <summary> /// <summary>
@@ -399,14 +205,12 @@ namespace SabreTools
/// <param name="basepath">Path the represents the parent directory</param> /// <param name="basepath">Path the represents the parent directory</param>
/// <param name="parent">Parent game to be used</param> /// <param name="parent">Parent game to be used</param>
/// <param name="datdata">DatData object with output information</param> /// <param name="datdata">DatData object with output information</param>
/// <param name="lastparent">Last known parent game name</param> private void ProcessFile(string item, string basepath, string parent, Dat datdata)
/// <returns>New last known parent game name</returns>
private string ProcessFile(string item, string basepath, string parent, Dat datdata, string lastparent)
{ {
_logger.Log(Path.GetFileName(item) + " treated like a file"); _logger.Log(Path.GetFileName(item) + " treated like a file");
Rom rom = FileTools.GetSingleFileInfo(item, _noMD5, _noSHA1); Rom rom = FileTools.GetSingleFileInfo(item, _noMD5, _noSHA1);
return ProcessFileHelper(item, rom, basepath, parent, datdata, lastparent); ProcessFileHelper(item, rom, basepath, parent, datdata);
} }
/// <summary> /// <summary>
@@ -417,9 +221,7 @@ namespace SabreTools
/// <param name="basepath">Path the represents the parent directory</param> /// <param name="basepath">Path the represents the parent directory</param>
/// <param name="parent">Parent game to be used</param> /// <param name="parent">Parent game to be used</param>
/// <param name="datdata">DatData object with output information</param> /// <param name="datdata">DatData object with output information</param>
/// <param name="lastparent">Last known parent game name</param> private void ProcessFileHelper(string item, Rom rom, string basepath, string parent, Dat datdata)
/// <returns>New last known parent game name</returns>
private string ProcessFileHelper(string item, Rom rom, string basepath, string parent, Dat datdata, string lastparent)
{ {
try try
{ {
@@ -469,25 +271,14 @@ namespace SabreTools
rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString()); rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString());
rom.Name = actualitem; rom.Name = actualitem;
string key = rom.HashData.Size + "-" + rom.HashData.CRC; _datdata.Files["null"].Add(rom);
if (_datdata.Files.ContainsKey(key))
{
_datdata.Files[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
_datdata.Files.Add(key, temp);
}
_logger.User("File added: " + actualitem + Environment.NewLine);
return rom.Machine.Name; _logger.User("File added: " + actualitem + Environment.NewLine);
} }
catch (IOException ex) catch (IOException ex)
{ {
_logger.Error(ex.ToString()); _logger.Error(ex.ToString());
return null; return;
} }
} }
} }