claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DatFile] Make filtering a separate step

Browse Source 
Making filter "optional" means that it can be done with multithreading, thus reducing the time that any given DAT takes to parse in most cases and improving the actual filtering in the process.
This commit is contained in:
Matt Nadareski
2017-03-17 23:44:22 -07:00
parent 0ebcd51cf9
commit 3f6553832b
5 changed files with 206 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
SabreTools.Helper/Dats/Partials/DatFile.ConvertUpdate.cs
View File

@@ -111,22 +111,17 @@ namespace SabreTools.Helper.Dats
MergeRoms = MergeRoms,
};
datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single,
root, true, clean, descAsName);
datHeaders[i].Parse(input.Split('¬')[0], i, 0, splitType, true, clean, descAsName);
});
Globals.Logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
Globals.Logger.User("Populating internal DAT");
Parallel.For(0, inputs.Count,
Globals.ParallelOptions,
i =>
Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
{
// Get the list of keys from the DAT
List<string> keys = datHeaders[i].Keys.ToList();
Parallel.ForEach(keys,
Globals.ParallelOptions,
key =>
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Add everything from the key to the internal DAT
AddRange(key, datHeaders[i][key]);
@@ -142,6 +137,9 @@ namespace SabreTools.Helper.Dats
datHeaders[i].Delete();
});
// Now that we have a merged DAT, filter it
Filter(filter, single, trim, root);
Globals.Logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
return datHeaders.ToList();
@@ -532,52 +530,49 @@ namespace SabreTools.Helper.Dats
public void Update(List<string> inputFileNames, string outDir, bool clean, bool descAsName, Filter filter,
SplitType splitType, bool trim, bool single, string root)
{
Parallel.ForEach(inputFileNames,
Globals.ParallelOptions,
inputFileName =>
Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName =>
{
// Clean the input string
if (inputFileName != "")
{
// Clean the input string
if (inputFileName != "")
{
inputFileName = Path.GetFullPath(inputFileName);
}
inputFileName = Path.GetFullPath(inputFileName);
}
if (File.Exists(inputFileName))
if (File.Exists(inputFileName))
{
DatFile innerDatdata = new DatFile(this);
Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
innerDatdata.Parse(inputFileName, 0, 0, splitType, true, clean, descAsName,
keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
innerDatdata.Filter(filter, trim, single, root);
// Try to output the file
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), overwrite: (outDir != ""));
}
else if (Directory.Exists(inputFileName))
{
inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(subFiles, Globals.ParallelOptions, file =>
{
Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
DatFile innerDatdata = new DatFile(this);
Globals.Logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
innerDatdata.Parse(inputFileName, 0, 0, filter, splitType, trim, single,
root, true, clean, descAsName,
innerDatdata.Parse(file, 0, 0, splitType, true, clean, descAsName,
keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
innerDatdata.Filter(filter, trim, single, root);
// Try to output the file
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), overwrite: (outDir != ""));
}
else if (Directory.Exists(inputFileName))
{
inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories),
Globals.ParallelOptions,
file =>
{
Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
DatFile innerDatdata = new DatFile(this);
innerDatdata.Parse(file, 0, 0, filter, splitType,
trim, single, root, true, clean, descAsName,
keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
// Try to output the file
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)),
overwrite: (outDir != ""));
});
}
else
{
Globals.Logger.Error("I'm sorry but " + inputFileName + " doesn't exist!");
return;
}
});
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)),
overwrite: (outDir != ""));
});
}
else
{
Globals.Logger.Error("I'm sorry but " + inputFileName + " doesn't exist!");
return;
}
});
}
#endregion

83
SabreTools.Helper/Dats/Partials/DatFile.Bucketing.cs → SabreTools.Helper/Dats/Partials/DatFile.Manipulate.cs
View File

@@ -6,13 +6,19 @@ using System.Web;
using SabreTools.Helper.Data;
#if MONO
using System.IO;
#else
using Alphaleonis.Win32.Filesystem;
#endif
namespace SabreTools.Helper.Dats
{
public partial class DatFile
{
#region Instance Methods
#region Instance Methods
#region Bucketing [MODULAR DONE]
#region Bucketing [MODULAR DONE]
/// <summary>
/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
@@ -151,9 +157,62 @@ namespace SabreTools.Helper.Dats
_files = sortable;
}
#endregion
#endregion
#region Merging/Splitting Methods [MODULAR DONE]
#region Filtering
/// <summary>
/// Filter a DAT based on input parameters and modify the items
/// </summary>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
public void Filter(Filter filter, bool single, bool trim, string root)
{
// Loop over every key in the dictionary
List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// For every item in the current key
List<DatItem> items = this[key];
List<DatItem> newitems = new List<DatItem>();
Parallel.ForEach(items, Globals.ParallelOptions, item =>
{
// If the rom passes the filter, include it
if (filter.ItemPasses(item))
{
// If we are in single game mode, rename all games
if (single)
{
item.Machine.Name = "!";
}
// If we are in NTFS trim mode, trim the game name
if (trim)
{
// Windows max name length is 260
int usableLength = 260 - item.Machine.Name.Length - root.Length;
if (item.Name.Length > usableLength)
{
string ext = Path.GetExtension(item.Name);
item.Name = item.Name.Substring(0, usableLength - ext.Length);
item.Name += ext;
}
}
newitems.Add(item);
}
});
Remove(key);
AddRange(key, newitems);
});
}
#endregion
#region Merging/Splitting Methods [MODULAR DONE]
/// <summary>
/// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets
@@ -247,9 +306,9 @@ namespace SabreTools.Helper.Dats
RemoveTagsFromChild();
}
#endregion
#endregion
#region Merging/Splitting Helper Methods [MODULAR DONE]
#region Merging/Splitting Helper Methods [MODULAR DONE]
/// <summary>
/// Use romof tags to add roms to the children
@@ -830,13 +889,13 @@ namespace SabreTools.Helper.Dats
}
}
#endregion
#endregion
#endregion // Instance Methods
#endregion // Instance Methods
#region Static Methods
#region Static Methods
#region Bucketing [MODULAR DONE]
#region Bucketing [MODULAR DONE]
/// <summary>
/// Take an arbitrarily ordered List and return a Dictionary sorted by Game
@@ -893,8 +952,8 @@ namespace SabreTools.Helper.Dats
return sortable;
}
#endregion
#endregion
#endregion // Static Methods
#endregion // Static Methods
}
}

345
SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs
View File

@@ -34,11 +34,9 @@ namespace SabreTools.Helper.Dats
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param>
public void Parse(string filename, int sysid, int srcid,
bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false)
public void Parse(string filename, int sysid, int srcid, bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false)
{
Parse(filename, sysid, srcid, new Filter(), SplitType.None, false, false, "",
keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags);
Parse(filename, sysid, srcid, SplitType.None, keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags);
}
/// <summary>
@@ -46,12 +44,8 @@ namespace SabreTools.Helper.Dats
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="srcid">Source ID for the DAT</param>>
/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -63,14 +57,8 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
SplitType splitType,
bool trim,
bool single,
string root,
// Miscellaneous
bool keep = false,
@@ -103,44 +91,44 @@ namespace SabreTools.Helper.Dats
switch (FileTools.GetDatFormat(filename))
{
case DatFormat.AttractMode:
ParseAttractMode(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
ParseAttractMode(filename, sysid, srcid, keep, clean, descAsName);
break;
case DatFormat.ClrMamePro:
case DatFormat.DOSCenter:
ParseCMP(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
ParseCMP(filename, sysid, srcid, keep, clean, descAsName);
break;
case DatFormat.CSV:
ParseCSVTSV(filename, sysid, srcid, ',', filter, trim, single, root, keep, clean, descAsName);
ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, descAsName);
break;
case DatFormat.Logiqx:
case DatFormat.OfflineList:
case DatFormat.SabreDat:
case DatFormat.SoftwareList:
ParseGenericXML(filename, sysid, srcid, filter, trim, single, root, keep, clean, descAsName);
ParseGenericXML(filename, sysid, srcid, keep, clean, descAsName);
break;
case DatFormat.RedumpMD5:
ParseRedumpMD5(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpMD5(filename, sysid, srcid, clean);
break;
case DatFormat.RedumpSFV:
ParseRedumpSFV(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpSFV(filename, sysid, srcid, clean);
break;
case DatFormat.RedumpSHA1:
ParseRedumpSHA1(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpSHA1(filename, sysid, srcid, clean);
break;
case DatFormat.RedumpSHA256:
ParseRedumpSHA256(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpSHA256(filename, sysid, srcid, clean);
break;
case DatFormat.RedumpSHA384:
ParseRedumpSHA384(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpSHA384(filename, sysid, srcid, clean);
break;
case DatFormat.RedumpSHA512:
ParseRedumpSHA512(filename, sysid, srcid, filter, trim, single, root, clean);
ParseRedumpSHA512(filename, sysid, srcid, clean);
break;
case DatFormat.RomCenter:
ParseRC(filename, sysid, srcid, filter, trim, single, root, clean, descAsName);
ParseRC(filename, sysid, srcid, clean, descAsName);
break;
case DatFormat.TSV:
ParseCSVTSV(filename, sysid, srcid, '\t', filter, trim, single, root, keep, clean, descAsName);
ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, descAsName);
break;
default:
return;
@@ -198,10 +186,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -211,14 +195,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool keep,
bool clean,
@@ -277,7 +253,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -289,10 +265,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
@@ -302,14 +274,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool keep,
bool clean,
@@ -421,7 +385,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the sample
key = "";
ParseAddHelper(item, filter, trim, single, root, clean, out key);
ParseAddHelper(item, clean, out key);
continue;
}
@@ -506,7 +470,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the rom
key = "";
ParseAddHelper(item, filter, trim, single, root, clean, out key);
ParseAddHelper(item, clean, out key);
continue;
}
@@ -719,7 +683,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the rom
key = "";
ParseAddHelper(item, filter, trim, single, root, clean, out key);
ParseAddHelper(item, clean, out key);
}
// If the line is anything but a rom or disk and we're in a block
@@ -904,10 +868,6 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="delim">Delimiter for parsing individual lines</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
@@ -918,14 +878,6 @@ namespace SabreTools.Helper.Dats
int srcid,
char delim,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool keep,
bool clean,
@@ -1164,7 +1116,7 @@ namespace SabreTools.Helper.Dats
},
};
ParseAddHelper(archive, filter, trim, single, root, clean, out key);
ParseAddHelper(archive, clean, out key);
break;
case ItemType.BiosSet:
BiosSet biosset = new BiosSet()
@@ -1178,7 +1130,7 @@ namespace SabreTools.Helper.Dats
},
};
ParseAddHelper(biosset, filter, trim, single, root, clean, out key);
ParseAddHelper(biosset, clean, out key);
break;
case ItemType.Disk:
Disk disk = new Disk()
@@ -1199,7 +1151,7 @@ namespace SabreTools.Helper.Dats
ItemStatus = status,
};
ParseAddHelper(disk, filter, trim, single, root, clean, out key);
ParseAddHelper(disk, clean, out key);
break;
case ItemType.Release:
Release release = new Release()
@@ -1213,7 +1165,7 @@ namespace SabreTools.Helper.Dats
},
};
ParseAddHelper(release, filter, trim, single, root, clean, out key);
ParseAddHelper(release, clean, out key);
break;
case ItemType.Rom:
Rom rom = new Rom()
@@ -1236,7 +1188,7 @@ namespace SabreTools.Helper.Dats
ItemStatus = status,
};
ParseAddHelper(rom, filter, trim, single, root, clean, out key);
ParseAddHelper(rom, clean, out key);
break;
case ItemType.Sample:
Sample sample = new Sample()
@@ -1250,7 +1202,7 @@ namespace SabreTools.Helper.Dats
},
};
ParseAddHelper(sample, filter, trim, single, root, clean, out key);
ParseAddHelper(sample, clean, out key);
break;
}
}
@@ -1262,10 +1214,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
@@ -1279,14 +1227,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool keep,
bool clean,
@@ -1325,7 +1265,7 @@ namespace SabreTools.Helper.Dats
Rom rom = new Rom("null", tempgame);
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out key);
ParseAddHelper(rom, clean, out key);
}
// Regardless, end the current folder
@@ -1871,7 +1811,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(olrom, filter, trim, single, root, clean, out key);
ParseAddHelper(olrom, clean, out key);
break;
// For Software List and MAME listxml only
@@ -1973,7 +1913,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(relrom, filter, trim, single, root, clean, out key);
ParseAddHelper(relrom, clean, out key);
subreader.Read();
break;
@@ -2016,7 +1956,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(biosrom, filter, trim, single, root, clean, out key);
ParseAddHelper(biosrom, clean, out key);
subreader.Read();
break;
@@ -2044,7 +1984,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(archiverom, filter, trim, single, root, clean, out key);
ParseAddHelper(archiverom, clean, out key);
subreader.Read();
break;
@@ -2072,7 +2012,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(samplerom, filter, trim, single, root, clean, out key);
ParseAddHelper(samplerom, clean, out key);
subreader.Read();
break;
@@ -2215,7 +2155,7 @@ namespace SabreTools.Helper.Dats
}
// Now process and add the rom
ParseAddHelper(inrom, filter, trim, single, root, clean, out key);
ParseAddHelper(inrom, clean, out key);
subreader.Read();
break;
@@ -2390,7 +2330,7 @@ namespace SabreTools.Helper.Dats
}
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out key);
ParseAddHelper(rom, clean, out key);
xtr.Read();
break;
@@ -2417,10 +2357,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpMD5(
// Standard Dat parsing
@@ -2428,14 +2364,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2464,7 +2392,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2476,10 +2404,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpSFV(
// Standard Dat parsing
@@ -2487,14 +2411,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2523,7 +2439,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2535,10 +2451,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpSHA1(
// Standard Dat parsing
@@ -2546,14 +2458,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2582,7 +2486,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2594,10 +2498,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpSHA256(
// Standard Dat parsing
@@ -2605,14 +2505,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2641,7 +2533,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2653,10 +2545,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpSHA384(
// Standard Dat parsing
@@ -2664,14 +2552,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2700,7 +2580,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2712,10 +2592,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
private void ParseRedumpSHA512(
// Standard Dat parsing
@@ -2723,14 +2599,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean)
{
@@ -2759,7 +2627,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
sr.Dispose();
@@ -2771,10 +2639,6 @@ namespace SabreTools.Helper.Dats
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
private void ParseRC(
@@ -2783,14 +2647,6 @@ namespace SabreTools.Helper.Dats
int sysid,
int srcid,
// Rom filtering
Filter filter,
// Rom renaming
bool trim,
bool single,
string root,
// Miscellaneous
bool clean,
bool descAsName)
@@ -2931,7 +2787,7 @@ namespace SabreTools.Helper.Dats
};
// Now process and add the rom
ParseAddHelper(rom, filter, trim, single, root, clean, out string key);
ParseAddHelper(rom, clean, out string key);
}
}
}
@@ -2943,11 +2799,9 @@ namespace SabreTools.Helper.Dats
/// Add a rom to the Dat after checking
/// </summary>
/// <param name="item">Item data to check against</param>
/// <param name="filter">Filter object for passing to the DatItem level</param>
/// <param name="trim">True if we are supposed to trim names to NTFS length, false otherwise</param>
/// <param name="single">True if all games should be replaced by '!', false otherwise</param>
/// <param name="root">String representing root directory to compare against for length calculation</param>
private void ParseAddHelper(DatItem item, Filter filter, bool trim, bool single, string root, bool clean, out string key)
/// <param name="clean">True if the names should be cleaned to WoD standards, false otherwise</param>
/// <param name="key">Output param containing the key for the item</param>
private void ParseAddHelper(DatItem item, bool clean, out string key)
{
key = "";
@@ -3046,80 +2900,57 @@ namespace SabreTools.Helper.Dats
item = itemDisk;
}
// If the rom passes the filter, include it
if (filter.ItemPasses(item))
// Get the key and add statistical data
switch (item.Type)
{
// If we are in single game mode, rename all games
if (single)
{
item.Machine.Name = "!";
}
case ItemType.Archive:
case ItemType.BiosSet:
case ItemType.Release:
case ItemType.Sample:
key = item.Type.ToString();
break;
case ItemType.Disk:
key = ((Disk)item).MD5;
// If we are in NTFS trim mode, trim the game name
if (trim)
{
// Windows max name length is 260
int usableLength = 260 - item.Machine.Name.Length - root.Length;
if (item.Name.Length > usableLength)
// Add statistical data
DiskCount += 1;
if (((Disk)item).ItemStatus != ItemStatus.Nodump)
{
string ext = Path.GetExtension(item.Name);
item.Name = item.Name.Substring(0, usableLength - ext.Length);
item.Name += ext;
TotalSize += 0;
MD5Count += (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1);
}
}
BaddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
case ItemType.Rom:
key = ((Rom)item).Size + "-" + ((Rom)item).CRC;
// Get the key and add statistical data
switch (item.Type)
{
case ItemType.Archive:
case ItemType.BiosSet:
case ItemType.Release:
case ItemType.Sample:
key = item.Type.ToString();
break;
case ItemType.Disk:
key = ((Disk)item).MD5;
// Add statistical data
DiskCount += 1;
if (((Disk)item).ItemStatus != ItemStatus.Nodump)
{
TotalSize += 0;
MD5Count += (String.IsNullOrEmpty(((Disk)item).MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(((Disk)item).SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(((Disk)item).SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(((Disk)item).SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(((Disk)item).SHA512) ? 0 : 1);
}
BaddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
case ItemType.Rom:
key = ((Rom)item).Size + "-" + ((Rom)item).CRC;
// Add statistical data
RomCount += 1;
if (((Rom)item).ItemStatus != ItemStatus.Nodump)
{
TotalSize += ((Rom)item).Size;
CRCCount += (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1);
MD5Count += (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1);
}
BaddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
default:
key = "default";
break;
}
// Add the item to the DAT
Add(key, item);
// Add statistical data
RomCount += 1;
if (((Rom)item).ItemStatus != ItemStatus.Nodump)
{
TotalSize += ((Rom)item).Size;
CRCCount += (String.IsNullOrEmpty(((Rom)item).CRC) ? 0 : 1);
MD5Count += (String.IsNullOrEmpty(((Rom)item).MD5) ? 0 : 1);
SHA1Count += (String.IsNullOrEmpty(((Rom)item).SHA1) ? 0 : 1);
SHA256Count += (String.IsNullOrEmpty(((Rom)item).SHA256) ? 0 : 1);
SHA384Count += (String.IsNullOrEmpty(((Rom)item).SHA384) ? 0 : 1);
SHA512Count += (String.IsNullOrEmpty(((Rom)item).SHA512) ? 0 : 1);
}
BaddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0);
NodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0);
break;
default:
key = "default";
break;
}
// Add the item to the DAT
Add(key, item);
}
#endregion

2
SabreTools.Helper/SabreTools.Helper.csproj
View File

@@ -114,7 +114,7 @@
<Compile Include="Data\Constants.cs" />
<Compile Include="Data\Flags.cs" />
<Compile Include="Data\Globals.cs" />
<Compile Include="Dats\Partials\DatFile.Bucketing.cs" />
<Compile Include="Dats\Partials\DatFile.Manipulate.cs" />
<Compile Include="Dats\Partials\DatFile.ConvertUpdate.cs" />
<Compile Include="Dats\Partials\DatFile.DFD.cs" />
<Compile Include="Dats\Partials\DatFile.Parsers.cs" />

8
SabreTools/Partials/SabreTools.Inits.cs
View File

@@ -325,7 +325,7 @@ namespace SabreTools
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
@@ -361,7 +361,7 @@ namespace SabreTools
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
@@ -728,7 +728,7 @@ namespace SabreTools
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
@@ -756,7 +756,7 @@ namespace SabreTools
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, new Filter(), splitType, false /* trim */, false /* single */, null /* root */, keep: true, useTags: true);
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
Globals.Logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));