claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ALL] Add flag to remove non-ASCII characters from items

Browse Source
This commit is contained in:
Matt Nadareski
2017-03-18 00:34:18 -07:00
parent 86cd96d077
commit 6263b5275f
7 changed files with 112 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
SabreTools.Helper/Dats/Partials/DatFile.Parsers.cs
View File

@@ -20,7 +20,7 @@ namespace SabreTools.Helper.Dats
{ {
public partial class DatFile public partial class DatFile
{ {
#region Parsing [MODULAR DONE, FOR NOW] #region Parsing
/// <summary> /// <summary>
/// Parse a DAT and return all found games and roms within /// Parse a DAT and return all found games and roms within
@@ -31,12 +31,15 @@ namespace SabreTools.Helper.Dats
/// <param name="datdata">The DatData object representing found roms to this point</param> /// <param name="datdata">The DatData object representing found roms to this point</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param> /// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param> /// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param>
public void Parse(string filename, int sysid, int srcid, bool keep = false, bool clean = false, bool descAsName = false, bool keepext = false, bool useTags = false) public void Parse(string filename, int sysid, int srcid, bool keep = false, bool clean = false,
bool remUnicode = false, bool descAsName = false, bool keepext = false, bool useTags = false)
{ {
Parse(filename, sysid, srcid, SplitType.None, keep: keep, clean: clean, descAsName: descAsName, keepext: keepext, useTags: useTags); Parse(filename, sysid, srcid, SplitType.None, keep: keep, clean: clean,
remUnicode: remUnicode, descAsName: descAsName, keepext: keepext, useTags: useTags);
} }
/// <summary> /// <summary>
@@ -48,6 +51,7 @@ namespace SabreTools.Helper.Dats
/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param> /// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param> /// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param> /// <param name="useTags">True if tags from the DAT should be used to merge the output, false otherwise (default)</param>
@@ -64,6 +68,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool keep = false, bool keep = false,
bool clean = false, bool clean = false,
bool remUnicode = false,
bool descAsName = false, bool descAsName = false,
bool keepext = false, bool keepext = false,
bool useTags = false) bool useTags = false)
@@ -92,44 +97,44 @@ namespace SabreTools.Helper.Dats
switch (FileTools.GetDatFormat(filename)) switch (FileTools.GetDatFormat(filename))
{ {
case DatFormat.AttractMode: case DatFormat.AttractMode:
ParseAttractMode(filename, sysid, srcid, keep, clean, descAsName); ParseAttractMode(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
break; break;
case DatFormat.ClrMamePro: case DatFormat.ClrMamePro:
case DatFormat.DOSCenter: case DatFormat.DOSCenter:
ParseCMP(filename, sysid, srcid, keep, clean, descAsName); ParseCMP(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
break; break;
case DatFormat.CSV: case DatFormat.CSV:
ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, descAsName); ParseCSVTSV(filename, sysid, srcid, ',', keep, clean, remUnicode, descAsName);
break; break;
case DatFormat.Logiqx: case DatFormat.Logiqx:
case DatFormat.OfflineList: case DatFormat.OfflineList:
case DatFormat.SabreDat: case DatFormat.SabreDat:
case DatFormat.SoftwareList: case DatFormat.SoftwareList:
ParseGenericXML(filename, sysid, srcid, keep, clean, descAsName); ParseGenericXML(filename, sysid, srcid, keep, clean, remUnicode, descAsName);
break; break;
case DatFormat.RedumpMD5: case DatFormat.RedumpMD5:
ParseRedumpMD5(filename, sysid, srcid, clean); ParseRedumpMD5(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RedumpSFV: case DatFormat.RedumpSFV:
ParseRedumpSFV(filename, sysid, srcid, clean); ParseRedumpSFV(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RedumpSHA1: case DatFormat.RedumpSHA1:
ParseRedumpSHA1(filename, sysid, srcid, clean); ParseRedumpSHA1(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RedumpSHA256: case DatFormat.RedumpSHA256:
ParseRedumpSHA256(filename, sysid, srcid, clean); ParseRedumpSHA256(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RedumpSHA384: case DatFormat.RedumpSHA384:
ParseRedumpSHA384(filename, sysid, srcid, clean); ParseRedumpSHA384(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RedumpSHA512: case DatFormat.RedumpSHA512:
ParseRedumpSHA512(filename, sysid, srcid, clean); ParseRedumpSHA512(filename, sysid, srcid, clean, remUnicode);
break; break;
case DatFormat.RomCenter: case DatFormat.RomCenter:
ParseRC(filename, sysid, srcid, clean, descAsName); ParseRC(filename, sysid, srcid, clean, remUnicode, descAsName);
break; break;
case DatFormat.TSV: case DatFormat.TSV:
ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, descAsName); ParseCSVTSV(filename, sysid, srcid, '\t', keep, clean, remUnicode, descAsName);
break; break;
default: default:
return; return;
@@ -189,6 +194,7 @@ namespace SabreTools.Helper.Dats
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
private void ParseAttractMode( private void ParseAttractMode(
// Standard Dat parsing // Standard Dat parsing
@@ -199,6 +205,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool keep, bool keep,
bool clean, bool clean,
bool remUnicode,
bool descAsName) bool descAsName)
{ {
// Open a file reader // Open a file reader
@@ -254,7 +261,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -268,6 +275,7 @@ namespace SabreTools.Helper.Dats
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
private void ParseCMP( private void ParseCMP(
// Standard Dat parsing // Standard Dat parsing
@@ -278,6 +286,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool keep, bool keep,
bool clean, bool clean,
bool remUnicode,
bool descAsName) bool descAsName)
{ {
// Open a file reader // Open a file reader
@@ -386,7 +395,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the sample // Now process and add the sample
key = ""; key = "";
ParseAddHelper(item, clean, out key); ParseAddHelper(item, clean, remUnicode, out key);
continue; continue;
} }
@@ -471,7 +480,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the rom // Now process and add the rom
key = ""; key = "";
ParseAddHelper(item, clean, out key); ParseAddHelper(item, clean, remUnicode, out key);
continue; continue;
} }
@@ -684,7 +693,7 @@ namespace SabreTools.Helper.Dats
// Now process and add the rom // Now process and add the rom
key = ""; key = "";
ParseAddHelper(item, clean, out key); ParseAddHelper(item, clean, remUnicode, out key);
} }
// If the line is anything but a rom or disk and we're in a block // If the line is anything but a rom or disk and we're in a block
@@ -871,6 +880,7 @@ namespace SabreTools.Helper.Dats
/// <param name="delim">Delimiter for parsing individual lines</param> /// <param name="delim">Delimiter for parsing individual lines</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param> /// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
private void ParseCSVTSV( private void ParseCSVTSV(
// Standard Dat parsing // Standard Dat parsing
@@ -882,6 +892,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool keep, bool keep,
bool clean, bool clean,
bool remUnicode,
bool descAsName) bool descAsName)
{ {
// Open a file reader // Open a file reader
@@ -1117,7 +1128,7 @@ namespace SabreTools.Helper.Dats
}, },
}; };
ParseAddHelper(archive, clean, out key); ParseAddHelper(archive, clean, remUnicode, out key);
break; break;
case ItemType.BiosSet: case ItemType.BiosSet:
BiosSet biosset = new BiosSet() BiosSet biosset = new BiosSet()
@@ -1131,7 +1142,7 @@ namespace SabreTools.Helper.Dats
}, },
}; };
ParseAddHelper(biosset, clean, out key); ParseAddHelper(biosset, clean, remUnicode, out key);
break; break;
case ItemType.Disk: case ItemType.Disk:
Disk disk = new Disk() Disk disk = new Disk()
@@ -1152,7 +1163,7 @@ namespace SabreTools.Helper.Dats
ItemStatus = status, ItemStatus = status,
}; };
ParseAddHelper(disk, clean, out key); ParseAddHelper(disk, clean, remUnicode, out key);
break; break;
case ItemType.Release: case ItemType.Release:
Release release = new Release() Release release = new Release()
@@ -1166,7 +1177,7 @@ namespace SabreTools.Helper.Dats
}, },
}; };
ParseAddHelper(release, clean, out key); ParseAddHelper(release, clean, remUnicode, out key);
break; break;
case ItemType.Rom: case ItemType.Rom:
Rom rom = new Rom() Rom rom = new Rom()
@@ -1189,7 +1200,7 @@ namespace SabreTools.Helper.Dats
ItemStatus = status, ItemStatus = status,
}; };
ParseAddHelper(rom, clean, out key); ParseAddHelper(rom, clean, remUnicode, out key);
break; break;
case ItemType.Sample: case ItemType.Sample:
Sample sample = new Sample() Sample sample = new Sample()
@@ -1203,7 +1214,7 @@ namespace SabreTools.Helper.Dats
}, },
}; };
ParseAddHelper(sample, clean, out key); ParseAddHelper(sample, clean, remUnicode, out key);
break; break;
} }
} }
@@ -1217,6 +1228,7 @@ namespace SabreTools.Helper.Dats
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param> /// <param name="descAsName">True if SL XML names should be kept, false otherwise (default)</param>
/// <remrks> /// <remrks>
/// TODO: Software Lists - sharedfeat tag (read-in, write-out) /// TODO: Software Lists - sharedfeat tag (read-in, write-out)
@@ -1230,6 +1242,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool keep, bool keep,
bool clean, bool clean,
bool remUnicode,
bool descAsName) bool descAsName)
{ {
// Prepare all internal variables // Prepare all internal variables
@@ -1265,7 +1278,7 @@ namespace SabreTools.Helper.Dats
Rom rom = new Rom("null", tempgame); Rom rom = new Rom("null", tempgame);
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out key); ParseAddHelper(rom, clean, remUnicode, out key);
} }
// Regardless, end the current folder // Regardless, end the current folder
@@ -1811,7 +1824,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(olrom, clean, out key); ParseAddHelper(olrom, clean, remUnicode, out key);
break; break;
// For Software List and MAME listxml only // For Software List and MAME listxml only
@@ -1913,7 +1926,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(relrom, clean, out key); ParseAddHelper(relrom, clean, remUnicode, out key);
subreader.Read(); subreader.Read();
break; break;
@@ -1956,7 +1969,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(biosrom, clean, out key); ParseAddHelper(biosrom, clean, remUnicode, out key);
subreader.Read(); subreader.Read();
break; break;
@@ -1984,7 +1997,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(archiverom, clean, out key); ParseAddHelper(archiverom, clean, remUnicode, out key);
subreader.Read(); subreader.Read();
break; break;
@@ -2012,7 +2025,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(samplerom, clean, out key); ParseAddHelper(samplerom, clean, remUnicode, out key);
subreader.Read(); subreader.Read();
break; break;
@@ -2155,7 +2168,7 @@ namespace SabreTools.Helper.Dats
} }
// Now process and add the rom // Now process and add the rom
ParseAddHelper(inrom, clean, out key); ParseAddHelper(inrom, clean, remUnicode, out key);
subreader.Read(); subreader.Read();
break; break;
@@ -2330,7 +2343,7 @@ namespace SabreTools.Helper.Dats
} }
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out key); ParseAddHelper(rom, clean, remUnicode, out key);
xtr.Read(); xtr.Read();
break; break;
@@ -2358,6 +2371,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpMD5( private void ParseRedumpMD5(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2365,7 +2379,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2392,7 +2407,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2405,6 +2420,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpSFV( private void ParseRedumpSFV(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2412,7 +2428,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2439,7 +2456,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2452,6 +2469,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpSHA1( private void ParseRedumpSHA1(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2459,7 +2477,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2486,7 +2505,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2499,6 +2518,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpSHA256( private void ParseRedumpSHA256(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2506,7 +2526,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2533,7 +2554,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2546,6 +2567,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpSHA384( private void ParseRedumpSHA384(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2553,7 +2575,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2580,7 +2603,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2593,6 +2616,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
private void ParseRedumpSHA512( private void ParseRedumpSHA512(
// Standard Dat parsing // Standard Dat parsing
string filename, string filename,
@@ -2600,7 +2624,8 @@ namespace SabreTools.Helper.Dats
int srcid, int srcid,
// Miscellaneous // Miscellaneous
bool clean) bool clean,
bool remUnicode)
{ {
// Open a file reader // Open a file reader
Encoding enc = Style.GetEncoding(filename); Encoding enc = Style.GetEncoding(filename);
@@ -2627,7 +2652,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
sr.Dispose(); sr.Dispose();
@@ -2640,6 +2665,7 @@ namespace SabreTools.Helper.Dats
/// <param name="sysid">System ID for the DAT</param> /// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param> /// <param name="srcid">Source ID for the DAT</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param> /// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
private void ParseRC( private void ParseRC(
// Standard Dat parsing // Standard Dat parsing
@@ -2649,6 +2675,7 @@ namespace SabreTools.Helper.Dats
// Miscellaneous // Miscellaneous
bool clean, bool clean,
bool remUnicode,
bool descAsName) bool descAsName)
{ {
// Open a file reader // Open a file reader
@@ -2787,7 +2814,7 @@ namespace SabreTools.Helper.Dats
}; };
// Now process and add the rom // Now process and add the rom
ParseAddHelper(rom, clean, out string key); ParseAddHelper(rom, clean, remUnicode, out string key);
} }
} }
} }
@@ -2800,8 +2827,9 @@ namespace SabreTools.Helper.Dats
/// </summary> /// </summary>
/// <param name="item">Item data to check against</param> /// <param name="item">Item data to check against</param>
/// <param name="clean">True if the names should be cleaned to WoD standards, false otherwise</param> /// <param name="clean">True if the names should be cleaned to WoD standards, false otherwise</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="key">Output param containing the key for the item</param> /// <param name="key">Output param containing the key for the item</param>
private void ParseAddHelper(DatItem item, bool clean, out string key) private void ParseAddHelper(DatItem item, bool clean, bool remUnicode, out string key)
{ {
key = ""; key = "";
@@ -2822,6 +2850,14 @@ namespace SabreTools.Helper.Dats
// If we're in cleaning mode, sanitize the game name // If we're in cleaning mode, sanitize the game name
item.Machine.Name = (clean ? Style.CleanGameName(item.Machine.Name) : item.Machine.Name); item.Machine.Name = (clean ? Style.CleanGameName(item.Machine.Name) : item.Machine.Name);
// If we're stripping unicode characters, do so from all relevant things
if (remUnicode)
{
item.Name = Style.RemoveUnicodeCharacters(item.Name);
item.Machine.Name = Style.RemoveUnicodeCharacters(item.Machine.Name);
item.Machine.Description = Style.RemoveUnicodeCharacters(item.Machine.Description);
}
// If we have a Rom or a Disk, clean the hash data // If we have a Rom or a Disk, clean the hash data
if (item.Type == ItemType.Rom) if (item.Type == ItemType.Rom)
{ {

4
SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
View File

@@ -34,10 +34,6 @@ namespace SabreTools.Helper.Dats
/// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param> /// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
/// <param name="overwrite">True if files should be overwritten (default), false if they should be renamed instead</param> /// <param name="overwrite">True if files should be overwritten (default), false if they should be renamed instead</param>
/// <returns>True if the DAT was written correctly, false otherwise</returns> /// <returns>True if the DAT was written correctly, false otherwise</returns>
/// <remarks>
/// The following features have been requested for file output:
/// - Have the ability to strip special (non-ASCII) characters from rom information
/// </remarks>
public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true) public bool WriteToFile(string outDir, bool norename = true, bool stats = false, bool ignoreblanks = false, bool overwrite = true)
{ {
// If there's nothing there, abort // If there's nothing there, abort

5
SabreTools.Helper/README.1ST
View File

@@ -919,6 +919,11 @@ Options:
Game names will be santitized to remove what the original WoD standards Game names will be santitized to remove what the original WoD standards
deemed as unneeded information, such as parenthized or bracketed strings deemed as unneeded information, such as parenthized or bracketed strings
-ru, --rem-uni Remove unicode characters from names
By default, the character set from the original file(s) will be used for
item naming. This flag removes all Unicode characters from the item names,
machine names, and machine descriptions.
-rmd5, --rem-md5 Remove MD5 hashes from the output -rmd5, --rem-md5 Remove MD5 hashes from the output
By default, all available hashes will be written out to the DAT. This will By default, all available hashes will be written out to the DAT. This will
remove all MD5 hashes from the output file(s). remove all MD5 hashes from the output file(s).

10
SabreTools.Helper/Tools/Style.cs
View File

@@ -454,6 +454,16 @@ namespace SabreTools.Helper.Tools
return new string(s.Where(c => !invalidPath.Contains(c)).ToArray()); return new string(s.Where(c => !invalidPath.Contains(c)).ToArray());
} }
/// <summary>
/// Remove all unicode-specific chars from a string
/// </summary>
/// <param name="s">Input string to clean</param>
/// <returns>Cleaned string</returns>
public static string RemoveUnicodeCharacters(string s)
{
return new string(s.Where(c => c > 255).ToArray());
}
/// <summary> /// <summary>
/// Split a line as if it were a CMP rom line /// Split a line as if it were a CMP rom line
/// </summary> /// </summary>

5
SabreTools/Partials/SabreTools.Help.cs
View File

@@ -1005,6 +1005,11 @@ namespace SabreTools
"Clean game names according to WoD standards", "Clean game names according to WoD standards",
FeatureType.Flag, FeatureType.Flag,
null)); null));
update.AddFeature("rem-uni", new Feature(
new List<string>() { "-ru", "--rem-uni" },
"Remove unicode characters from names",
FeatureType.Flag,
null));
update.AddFeature("rem-md5", new Feature( update.AddFeature("rem-md5", new Feature(
new List<string>() { "-rmd5", "--rem-md5" }, new List<string>() { "-rmd5", "--rem-md5" },
"Remove MD5 hashes from the output", "Remove MD5 hashes from the output",

2
SabreTools/Partials/SabreTools.Inits.cs
View File

@@ -472,6 +472,7 @@ namespace SabreTools
/// /* Output DAT info */ /// /* Output DAT info */
/// <param name="outDir">Optional param for output directory</param> /// <param name="outDir">Optional param for output directory</param>
/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param> /// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
/// <param name="dedup">True to dedupe the roms in the DAT, false otherwise (default)</param> /// <param name="dedup">True to dedupe the roms in the DAT, false otherwise (default)</param>
/// <param name="stripHash">StripHash that represents the hash(es) that you want to remove from the output</param> /// <param name="stripHash">StripHash that represents the hash(es) that you want to remove from the output</param>
@@ -529,6 +530,7 @@ namespace SabreTools
/* Output DAT info */ /* Output DAT info */
string outDir, string outDir,
bool clean, bool clean,
bool remUnicode,
bool descAsName, bool descAsName,
bool dedup, bool dedup,
Hash stripHash) Hash stripHash)

7
SabreTools/SabreTools.cs
View File

@@ -112,6 +112,7 @@ namespace SabreTools
quotes = false, quotes = false,
remext = false, remext = false,
removeDateFromAutomaticName = false, removeDateFromAutomaticName = false,
removeUnicode = false,
romba = false, romba = false,
showBaddumpColumn = false, showBaddumpColumn = false,
showNodumpColumn = false, showNodumpColumn = false,
@@ -539,6 +540,10 @@ namespace SabreTools
case "--rem-sha512": case "--rem-sha512":
stripHash |= Hash.SHA512; stripHash |= Hash.SHA512;
break; break;
case "-ru":
case "--rem-uni":
removeUnicode = true;
break;
case "-run": case "-run":
case "--runnable": case "--runnable":
filter.Runnable = true; filter.Runnable = true;
@@ -1271,7 +1276,7 @@ namespace SabreTools
InitUpdate(inputs, filename, name, description, rootdir, category, version, date, author, email, homepage, url, comment, header, InitUpdate(inputs, filename, name, description, rootdir, category, version, date, author, email, homepage, url, comment, header,
superdat, forcemerge, forcend, forcepack, excludeOf, datFormat, usegame, prefix, postfix, quotes, repext, addext, remext, superdat, forcemerge, forcend, forcepack, excludeOf, datFormat, usegame, prefix, postfix, quotes, repext, addext, remext,
datPrefix, romba, merge, diffMode, inplace, skip, removeDateFromAutomaticName, filter, oneGameOneRegion, regions, datPrefix, romba, merge, diffMode, inplace, skip, removeDateFromAutomaticName, filter, oneGameOneRegion, regions,
splitType, trim, single, root, outDir, cleanGameNames, descAsName, dedup, stripHash); splitType, trim, single, root, outDir, cleanGameNames, removeUnicode, descAsName, dedup, stripHash);
} }
// If we're using the verifier // If we're using the verifier