claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DatFile] Parallel hunting

Browse Source
This commit is contained in:
Matt Nadareski
2017-03-18 21:26:50 -07:00
parent c9c532776b
commit 23c5d9601d
8 changed files with 381 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
SabreTools.Helper/Dats/DatFile.cs
View File

@@ -1,5 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Threading.Tasks;
using SabreTools.Helper.Data;
@@ -460,10 +460,10 @@ namespace SabreTools.Helper.Dats
lock (_files)
{
int count = 0;
foreach (string key in _files.Keys)
Parallel.ForEach(_files.Keys, Globals.ParallelOptions, key =>
{
count += _files[key].Count;
}
});
return count;
}

214
SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs
View File

@@ -221,51 +221,57 @@ namespace SabreTools.Helper.Dats
// Now, loop through the dictionary and populate the correct DATs
start = DateTime.Now;
Globals.Logger.User("Populating all output DATs");
List<string> keys = Keys.ToList();
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = DatItem.Merge(this[key]);
List<DatItem> items = DatItem.Merge(this[key]);
if (roms != null && roms.Count > 0)
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
{
foreach (DatItem rom in roms)
return;
}
// Loop through and add the items correctly
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
// No duplicates
if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
{
// No duplicates
if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
if ((item.Dupe & DupeType.Internal) != 0)
{
if ((rom.Dupe & DupeType.Internal) != 0)
// Individual DATs that are output
if ((diff & DiffMode.Individuals) != 0)
{
// Individual DATs that are output
if ((diff & DiffMode.Individuals) != 0)
{
outDats[rom.SystemID].Add(key, rom);
}
// Merged no-duplicates DAT
if ((diff & DiffMode.NoDupes) != 0)
{
DatItem newrom = rom;
newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
outerDiffData.Add(key, newrom);
}
outDats[item.SystemID].Add(key, item);
}
}
// Duplicates only
if ((diff & DiffMode.Dupes) != 0)
{
if ((rom.Dupe & DupeType.External) != 0)
// Merged no-duplicates DAT
if ((diff & DiffMode.NoDupes) != 0)
{
DatItem newrom = rom;
DatItem newrom = item;
newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
dupeData.Add(key, newrom);
outerDiffData.Add(key, newrom);
}
}
}
}
}
// Duplicates only
if ((diff & DiffMode.Dupes) != 0)
{
if ((item.Dupe & DupeType.External) != 0)
{
DatItem newrom = item;
newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
dupeData.Add(key, newrom);
}
}
});
});
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Finally, loop through and output each of the DATs
@@ -287,7 +293,7 @@ namespace SabreTools.Helper.Dats
// Output the individual (a-b) DATs
if ((diff & DiffMode.Individuals) != 0)
{
for (int j = 0; j < inputs.Count; j++)
Parallel.For(0, inputs.Count, j =>
{
// If we have an output directory set, replace the path
string[] split = inputs[j].Split('¬');
@@ -297,7 +303,7 @@ namespace SabreTools.Helper.Dats
// Try to output the file
outDats[j].WriteToFile(path);
}
});
}
Globals.Logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
}
@@ -353,31 +359,36 @@ namespace SabreTools.Helper.Dats
Globals.Logger.User("Populating all output DATs");
List<string> keys = Keys.ToList();
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = DatItem.Merge(this[key]);
List<DatItem> items = DatItem.Merge(this[key]);
if (roms != null && roms.Count > 0)
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
{
foreach (DatItem rom in roms)
{
// There's odd cases where there are items with System ID < 0. Skip them for now
if (rom.SystemID < 0)
{
Globals.Logger.Warning("Item found with a <0 SystemID: " + rom.Name);
continue;
}
outDats[rom.SystemID].Add(key, rom);
}
return;
}
}
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
// There's odd cases where there are items with System ID < 0. Skip them for now
if (item.SystemID < 0)
{
Globals.Logger.Warning("Item found with a <0 SystemID: " + item.Name);
return;
}
outDats[item.SystemID].Add(key, item);
});
});
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Finally, loop through and output each of the DATs
start = DateTime.Now;
Globals.Logger.User("Outputting all created DATs");
for (int j = (skip ? 1 : 0); j < inputs.Count; j++)
Parallel.For((skip ? 1 : 0), inputs.Count, j =>
{
// If we have an output directory set, replace the path
string path = "";
@@ -395,7 +406,8 @@ namespace SabreTools.Helper.Dats
// Try to output the file
outDats[j].WriteToFile(path);
}
});
Globals.Logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
}
@@ -411,103 +423,37 @@ namespace SabreTools.Helper.Dats
if (Type == "SuperDAT")
{
List<string> keys = Keys.ToList();
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> newroms = new List<DatItem>();
foreach (DatItem rom in this[key])
List<DatItem> items = this[key].ToList();
List<DatItem> newItems = new List<DatItem>();
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
DatItem newrom = rom;
string filename = inputs[newrom.SystemID].Split('¬')[0];
string rootpath = inputs[newrom.SystemID].Split('¬')[1];
DatItem newItem = item;
string filename = inputs[newItem.SystemID].Split('¬')[0];
string rootpath = inputs[newItem.SystemID].Split('¬')[1];
rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString());
filename = filename.Remove(0, rootpath.Length);
newrom.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
newItem.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+ Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+ newrom.Machine.Name;
newroms.Add(newrom);
}
this[key] = newroms;
}
+ newItem.Machine.Name;
lock (newItems)
{
newItems.Add(newItem);
}
});
Remove(key);
AddRange(key, newItems);
});
}
// Try to output the file
WriteToFile(outDir);
}
/// <summary>
/// Strip the given hash types from the DAT
/// </summary>
private void StripHashesFromItems()
{
// Output the logging statement
Globals.Logger.User("Stripping requested hashes");
// Now process all of the roms
List<string> keys = Keys.ToList();
for (int i = 0; i < keys.Count; i++)
{
List<DatItem> items = this[keys[i]];
for (int j = 0; j < items.Count; j++)
{
DatItem item = items[j];
if (item.Type == ItemType.Rom)
{
Rom rom = (Rom)item;
if ((StripHash & Hash.MD5) != 0)
{
rom.MD5 = null;
}
if ((StripHash & Hash.SHA1) != 0)
{
rom.SHA1 = null;
}
if ((StripHash & Hash.SHA256) != 0)
{
rom.SHA256 = null;
}
if ((StripHash & Hash.SHA384) != 0)
{
rom.SHA384 = null;
}
if ((StripHash & Hash.SHA512) != 0)
{
rom.SHA512 = null;
}
items[j] = rom;
}
else if (item.Type == ItemType.Disk)
{
Disk disk = (Disk)item;
if ((StripHash & Hash.MD5) != 0)
{
disk.MD5 = null;
}
if ((StripHash & Hash.SHA1) != 0)
{
disk.SHA1 = null;
}
if ((StripHash & Hash.SHA256) != 0)
{
disk.SHA256 = null;
}
if ((StripHash & Hash.SHA384) != 0)
{
disk.SHA384 = null;
}
if ((StripHash & Hash.SHA512) != 0)
{
disk.SHA512 = null;
}
items[j] = disk;
}
}
this[keys[i]] = items;
}
}
/// <summary>
/// Convert, update, and filter a DAT file or set of files using a base
/// </summary>

4
SabreTools.Helper/Dats/Partials/DatFile.DFD.cs
View File

@@ -224,13 +224,13 @@ namespace SabreTools.Helper.Dats
else
{
// First take care of the found items
foreach (Rom rom in extracted)
Parallel.ForEach(extracted, Globals.ParallelOptions, rom =>
{
PopulateFromDirProcessFileHelper(newItem,
rom,
basePath,
(Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item));
}
});
// Then, if we're looking for blanks, get all of the blank folders and add them
if (addBlanks)

333
SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs
View File

@@ -16,144 +16,144 @@ namespace SabreTools.Helper.Dats
{
public partial class DatFile
{
#region Instance Methods
#region Instance Methods
#region Bucketing
#region Bucketing
/// <summary>
/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
/// </summary>
/// <param name="bucketBy">SortedBy enum representing how to sort the individual items</param>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true)
{
// If we already have the right sorting, trust it
if (_sortedBy == bucketBy)
{
return;
}
// Set the sorted type
_sortedBy = bucketBy;
// Create the temporary dictionary to sort into
SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : ""));
// First do the initial sort of all of the roms
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = this[key];
// Now add each of the roms to their respective games
Parallel.ForEach(roms, Globals.ParallelOptions, rom =>
/// <summary>
/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
/// </summary>
/// <param name="bucketBy">SortedBy enum representing how to sort the individual items</param>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true)
{
string newkey = "";
// We want to get the key most appropriate for the given sorting type
switch (bucketBy)
// If we already have the right sorting, trust it
if (_sortedBy == bucketBy)
{
case SortedBy.CRC:
newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
break;
case SortedBy.Game:
newkey = (norename ? ""
: rom.SystemID.ToString().PadLeft(10, '0')
+ "-"
+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
+ (String.IsNullOrEmpty(rom.Machine.Name)
? "Default"
: rom.Machine.Name);
if (lower)
return;
}
// Set the sorted type
_sortedBy = bucketBy;
// Create the temporary dictionary to sort into
SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
Globals.Logger.User("Organizing roms by " + bucketBy +(mergeroms ? " and merging" : ""));
// First do the initial sort of all of the roms
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = this[key];
// Now add each of the roms to their respective games
Parallel.ForEach(roms, Globals.ParallelOptions, rom =>
{
string newkey = "";
// We want to get the key most appropriate for the given sorting type
switch (bucketBy)
{
newkey = newkey.ToLowerInvariant();
case SortedBy.CRC:
newkey = (rom.Type == ItemType.Rom ? ((Rom)rom).CRC : Constants.CRCZero);
break;
case SortedBy.Game:
newkey = (norename ? ""
: rom.SystemID.ToString().PadLeft(10, '0')
+ "-"
+ rom.SourceID.ToString().PadLeft(10, '0') + "-")
+ (String.IsNullOrEmpty(rom.Machine.Name)
? "Default"
: rom.Machine.Name);
if (lower)
{
newkey = newkey.ToLowerInvariant();
}
newkey = HttpUtility.HtmlEncode(newkey);
break;
case SortedBy.MD5:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).MD5
: (rom.Type == ItemType.Disk
? ((Disk)rom).MD5
: Constants.MD5Zero));
break;
case SortedBy.SHA1:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA1
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA1
: Constants.SHA1Zero));
break;
case SortedBy.SHA256:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA256
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA256
: Constants.SHA256Zero));
break;
case SortedBy.SHA384:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA384
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA384
: Constants.SHA384Zero));
break;
case SortedBy.SHA512:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA512
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA512
: Constants.SHA512Zero));
break;
}
newkey = HttpUtility.HtmlEncode(newkey);
break;
case SortedBy.MD5:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).MD5
: (rom.Type == ItemType.Disk
? ((Disk)rom).MD5
: Constants.MD5Zero));
break;
case SortedBy.SHA1:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA1
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA1
: Constants.SHA1Zero));
break;
case SortedBy.SHA256:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA256
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA256
: Constants.SHA256Zero));
break;
case SortedBy.SHA384:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA384
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA384
: Constants.SHA384Zero));
break;
case SortedBy.SHA512:
newkey = (rom.Type == ItemType.Rom
? ((Rom)rom).SHA512
: (rom.Type == ItemType.Disk
? ((Disk)rom).SHA512
: Constants.SHA512Zero));
break;
}
// Add the DatItem to the temp dictionary
lock (sortable)
{
if (!sortable.ContainsKey(newkey))
{
sortable.Add(newkey, new List<DatItem>());
}
sortable[newkey].Add(rom);
}
});
});
// Add the DatItem to the temp dictionary
lock (sortable)
// Now go through and sort all of the individual lists
keys = sortable.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
if (!sortable.ContainsKey(newkey))
// Get the possibly unsorted list
List<DatItem> sortedlist = sortable[key];
// Sort the list of items to be consistent
DatItem.Sort(ref sortedlist, false);
// If we're merging the roms, do so
if (mergeroms)
{
sortable.Add(newkey, new List<DatItem>());
sortedlist = DatItem.Merge(sortedlist);
}
sortable[newkey].Add(rom);
}
});
});
// Now go through and sort all of the individual lists
keys = sortable.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Get the possibly unsorted list
List<DatItem> sortedlist = sortable[key];
// Add the list back to the temp dictionary
lock (sortable)
{
sortable[key] = sortedlist;
}
});
// Sort the list of items to be consistent
DatItem.Sort(ref sortedlist, false);
// If we're merging the roms, do so
if (mergeroms)
{
sortedlist = DatItem.Merge(sortedlist);
// Now assign the dictionary back
_files = sortable;
}
// Add the list back to the temp dictionary
lock (sortable)
{
sortable[key] = sortedlist;
}
});
#endregion
// Now assign the dictionary back
_files = sortable;
}
#endregion
#region Filtering
#region Filtering
/// <summary>
/// Filter a DAT based on input parameters and modify the items
@@ -284,9 +284,90 @@ namespace SabreTools.Helper.Dats
}
}
#endregion
/// <summary>
/// Strip the given hash types from the DAT
/// </summary>
public void StripHashesFromItems()
{
// Output the logging statement
Globals.Logger.User("Stripping requested hashes");
#region Merging/Splitting Methods
// Now process all of the roms
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = this[key];
Parallel.For(0, items.Count, Globals.ParallelOptions, j =>
{
DatItem item = items[j];
if (item.Type == ItemType.Rom)
{
Rom rom = (Rom)item;
if ((StripHash & Hash.MD5) != 0)
{
rom.MD5 = null;
}
if ((StripHash & Hash.SHA1) != 0)
{
rom.SHA1 = null;
}
if ((StripHash & Hash.SHA256) != 0)
{
rom.SHA256 = null;
}
if ((StripHash & Hash.SHA384) != 0)
{
rom.SHA384 = null;
}
if ((StripHash & Hash.SHA512) != 0)
{
rom.SHA512 = null;
}
lock (items)
{
items[j] = rom;
}
}
else if (item.Type == ItemType.Disk)
{
Disk disk = (Disk)item;
if ((StripHash & Hash.MD5) != 0)
{
disk.MD5 = null;
}
if ((StripHash & Hash.SHA1) != 0)
{
disk.SHA1 = null;
}
if ((StripHash & Hash.SHA256) != 0)
{
disk.SHA256 = null;
}
if ((StripHash & Hash.SHA384) != 0)
{
disk.SHA384 = null;
}
if ((StripHash & Hash.SHA512) != 0)
{
disk.SHA512 = null;
}
lock (items)
{
items[j] = disk;
}
}
});
Remove(key);
AddRange(key, items);
});
}
#endregion
#region Merging/Splitting Methods
/// <summary>
/// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets
@@ -382,7 +463,7 @@ namespace SabreTools.Helper.Dats
#endregion
#region Merging/Splitting Helper Methods
#region Merging/Splitting Helper Methods
/// <summary>
/// Use romof tags to add roms to the children
@@ -965,6 +1046,6 @@ namespace SabreTools.Helper.Dats
#endregion
#endregion // Instance Methods
#endregion // Instance Methods
}
}

10
SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Helper.Data;
using SabreTools.Helper.Skippers;
@@ -126,15 +127,18 @@ namespace SabreTools.Helper.Dats
// Now loop through and get only directories from the input paths
List<string> directories = new List<string>();
foreach (string input in inputs)
Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
Globals.Logger.Verbose("Adding depot: '" + input + "'");
directories.Add(input);
lock (directories)
{
directories.Add(input);
}
}
}
});
// If we don't have any directories, we want to exit
if (directories.Count == 0)

125
SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using SabreTools.Helper.Data;
@@ -31,17 +32,23 @@ namespace SabreTools.Helper.Dats
{
// Make sure all of the extensions have a dot at the beginning
List<string> newExtA = new List<string>();
foreach (string s in extA)
Parallel.ForEach(extA, Globals.ParallelOptions, s =>
{
newExtA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
}
lock (newExtA)
{
newExtA.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
}
});
string newExtAString = string.Join(",", newExtA);
List<string> newExtB = new List<string>();
foreach (string s in extB)
Parallel.ForEach(extB, Globals.ParallelOptions, s =>
{
newExtB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
}
lock (newExtB)
{
newExtB.Add((s.StartsWith(".") ? s : "." + s).ToUpperInvariant());
}
});
string newExtBString = string.Join(",", newExtB);
// Set all of the appropriate outputs for each of the subsets
@@ -83,25 +90,27 @@ namespace SabreTools.Helper.Dats
}
// Now separate the roms accordingly
foreach (string key in Keys)
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
foreach (DatItem rom in this[key])
List<DatItem> items = this[key];
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
if (newExtA.Contains(Path.GetExtension(rom.Name.ToUpperInvariant())))
if (newExtA.Contains(Path.GetExtension(item.Name.ToUpperInvariant())))
{
datdataA.Add(key, rom);
datdataA.Add(key, item);
}
else if (newExtB.Contains(Path.GetExtension(rom.Name.ToUpperInvariant())))
else if (newExtB.Contains(Path.GetExtension(item.Name.ToUpperInvariant())))
{
datdataB.Add(key, rom);
datdataB.Add(key, item);
}
else
{
datdataA.Add(key, rom);
datdataB.Add(key, rom);
datdataA.Add(key, item);
datdataB.Add(key, item);
}
}
}
});
});
// Get the output directory
if (outDir != "")
@@ -304,65 +313,65 @@ namespace SabreTools.Helper.Dats
// Now populate each of the DAT objects in turn
List<string> keys = Keys.ToList();
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = this[key];
foreach (DatItem rom in roms)
List<DatItem> items = this[key];
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
// If the file is not a Rom or Disk, continue
if (rom.Type != ItemType.Disk && rom.Type != ItemType.Rom)
if (item.Type != ItemType.Disk && item.Type != ItemType.Rom)
{
continue;
return;
}
// If the file is a nodump
if ((rom.Type == ItemType.Rom && ((Rom)rom).ItemStatus == ItemStatus.Nodump)
|| (rom.Type == ItemType.Disk && ((Disk)rom).ItemStatus == ItemStatus.Nodump))
if ((item.Type == ItemType.Rom && ((Rom)item).ItemStatus == ItemStatus.Nodump)
|| (item.Type == ItemType.Disk && ((Disk)item).ItemStatus == ItemStatus.Nodump))
{
nodump.Add(key, rom);
nodump.Add(key, item);
}
// If the file has a SHA-512
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA512))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA512)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA512))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA512)))
{
sha512.Add(key, rom);
sha512.Add(key, item);
}
// If the file has a SHA-384
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA384))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA384)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA384))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA384)))
{
sha384.Add(key, rom);
sha384.Add(key, item);
}
// If the file has a SHA-256
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA256))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA256)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA256))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA256)))
{
sha256.Add(key, rom);
sha256.Add(key, item);
}
// If the file has a SHA-1
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA1))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA1)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1)))
{
sha1.Add(key, rom);
sha1.Add(key, item);
}
// If the file has no SHA-1 but has an MD5
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).MD5))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).MD5)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).MD5))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).MD5)))
{
md5.Add(key, rom);
md5.Add(key, item);
}
// If the file has no MD5 but a CRC
else if ((rom.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)rom).SHA1))
|| (rom.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)rom).SHA1)))
else if ((item.Type == ItemType.Rom && !String.IsNullOrEmpty(((Rom)item).SHA1))
|| (item.Type == ItemType.Disk && !String.IsNullOrEmpty(((Disk)item).SHA1)))
{
crc.Add(key, rom);
crc.Add(key, item);
}
else
{
other.Add(key, rom);
other.Add(key, item);
}
}
}
});
});
// Get the output directory
if (outDir != "")
@@ -415,7 +424,7 @@ namespace SabreTools.Helper.Dats
keys.Sort(SplitByLevelSort);
// Then, we loop over the games
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Here, the key is the name of the game to be used for comparison
if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key))
@@ -440,7 +449,7 @@ namespace SabreTools.Helper.Dats
// Then set the DAT name to be the parent directory name
tempDat.Name = Style.GetDirectoryName(key);
}
});
// Then we write the last DAT out since it would be skipped otherwise
SplitByLevelHelper(tempDat, outDir, shortname, basedat);
@@ -581,28 +590,28 @@ namespace SabreTools.Helper.Dats
// Now populate each of the DAT objects in turn
List<string> keys = Keys.ToList();
foreach (string key in keys)
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> roms = this[key];
foreach (DatItem rom in roms)
List<DatItem> items = this[key];
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
// If the file is a Rom
if (rom.Type == ItemType.Rom)
if (item.Type == ItemType.Rom)
{
romdat.Add(key, rom);
romdat.Add(key, item);
}
// If the file is a Disk
else if (rom.Type == ItemType.Disk)
else if (item.Type == ItemType.Disk)
{
diskdat.Add(key, rom);
diskdat.Add(key, item);
}
// If the file is a Sample
else if (rom.Type == ItemType.Sample)
else if (item.Type == ItemType.Sample)
{
sampledat.Add(key, rom);
sampledat.Add(key, item);
}
}
}
});
});
// Get the output directory
if (outDir != "")

14
SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using SabreTools.Helper.Data;
@@ -51,10 +52,11 @@ namespace SabreTools.Helper.Dats
}
// Loop through and add
foreach (string key in Keys)
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = this[key];
foreach (DatItem item in items)
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
switch (item.Type)
{
@@ -68,6 +70,8 @@ namespace SabreTools.Helper.Dats
MD5Count += (String.IsNullOrEmpty(disk.MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(disk.SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(disk.SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(disk.SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(disk.SHA512) ? 0 : 1);
BaddumpCount += (disk.ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (disk.ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
@@ -81,14 +85,16 @@ namespace SabreTools.Helper.Dats
MD5Count += (String.IsNullOrEmpty(rom.MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(rom.SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(rom.SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(rom.SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(rom.SHA512) ? 0 : 1);
BaddumpCount += (rom.ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (rom.ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
case ItemType.Sample:
break;
}
}
}
});
});
}
/// <summary>

6
SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using SabreTools.Helper.Data;
@@ -134,7 +135,8 @@ namespace SabreTools.Helper.Dats
List<string> keys = Keys.ToList();
keys.Sort(new NaturalComparer());
foreach (DatFormat datFormat in outfiles.Keys)
// Write out all required formats
Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat =>
{
string outfile = outfiles[datFormat];
@@ -229,7 +231,7 @@ namespace SabreTools.Helper.Dats
Globals.Logger.Verbose("File written!" + Environment.NewLine);
sw.Dispose();
fs.Dispose();
}
});
}
catch (Exception ex)
{