[DatFiles/] Create separate classes for each type of DAT

This DOES group some kinds of DAT due to their inherently similar nature. TSV and CSV are under the same "SeparatedValue" umbrella, and all of the SFV, MD5, SHA1, etc are under Hashfile because they're nearly identical. This is just the first stage change, making everything static and making them reference the DatFile separately.
This commit is contained in:
Matt Nadareski
2017-10-09 18:04:49 -07:00
parent 4eb36aa39f
commit 2209d0a13b
16 changed files with 6878 additions and 4594 deletions

View File

@@ -0,0 +1,612 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using SabreTools.Library.Data;
using SabreTools.Library.Items;
using SabreTools.Library.Tools;
#if MONO
using System.IO;
#else
using FileStream = System.IO.FileStream;
using StreamReader = System.IO.StreamReader;
using StreamWriter = System.IO.StreamWriter;
#endif
using NaturalSort;
namespace SabreTools.Library.DatFiles
{
/// <summary>
/// Represents parsing and writing of a value-separated DAT
/// </summary>
public class SeparatedValue
{
/// <summary>
/// Parse a character-separated value DAT and return all found games and roms within
/// </summary>
/// <param name="datFile">DatFile to populate with the read information</param>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="delim">Delimiter for parsing individual lines</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
public static void Parse(
DatFile datFile,
// Standard Dat parsing
string filename,
int sysid,
int srcid,
char delim,
// Miscellaneous
bool keep,
bool clean,
bool remUnicode)
{
// Open a file reader
Encoding enc = Style.GetEncoding(filename);
StreamReader sr = new StreamReader(FileTools.TryOpenRead(filename), enc);
// Create an empty list of columns to parse though
List<string> columns = new List<string>();
long linenum = -1;
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
linenum++;
// Parse the first line, getting types from the column names
if (linenum == 0)
{
string[] parsedColumns = line.Split(delim);
foreach (string parsed in parsedColumns)
{
switch (parsed.ToLowerInvariant().Trim('"'))
{
case "file":
case "filename":
case "file name":
columns.Add("DatFile.FileName");
break;
case "internal name":
columns.Add("DatFile.Name");
break;
case "description":
case "dat description":
columns.Add("DatFile.Description");
break;
case "game name":
case "game":
case "machine":
columns.Add("Machine.Name");
break;
case "game description":
columns.Add("Description");
break;
case "type":
columns.Add("DatItem.Type");
break;
case "rom":
case "romname":
case "rom name":
case "name":
columns.Add("Rom.Name");
break;
case "disk":
case "diskname":
case "disk name":
columns.Add("Disk.Name");
break;
case "size":
columns.Add("DatItem.Size");
break;
case "crc":
case "crc hash":
columns.Add("DatItem.CRC");
break;
case "md5":
case "md5 hash":
columns.Add("DatItem.MD5");
break;
case "sha1":
case "sha-1":
case "sha1 hash":
case "sha-1 hash":
columns.Add("DatItem.SHA1");
break;
case "sha256":
case "sha-256":
case "sha256 hash":
case "sha-256 hash":
columns.Add("DatItem.SHA256");
break;
case "sha384":
case "sha-384":
case "sha384 hash":
case "sha-384 hash":
columns.Add("DatItem.SHA384");
break;
case "sha512":
case "sha-512":
case "sha512 hash":
case "sha-512 hash":
columns.Add("DatItem.SHA512");
break;
case "nodump":
case "no dump":
case "status":
case "item status":
columns.Add("DatItem.Nodump");
break;
default:
columns.Add("INVALID");
break;
}
}
continue;
}
// Otherwise, we want to split the line and parse
string[] parsedLine = line.Split(delim);
// If the line doesn't have the correct number of columns, we log and skip
if (parsedLine.Length != columns.Count)
{
Globals.Logger.Warning("Malformed line found in '{0}' at line {1}", filename, linenum);
continue;
}
// Set the output item information
string machineName = null, machineDesc = null, name = null, crc = null, md5 = null, sha1 = null,
sha256 = null, sha384 = null, sha512 = null;
long size = -1;
ItemType itemType = ItemType.Rom;
ItemStatus status = ItemStatus.None;
// Now we loop through and get values for everything
for (int i = 0; i < columns.Count; i++)
{
string value = parsedLine[i].Trim('"');
switch (columns[i])
{
case "DatFile.FileName":
datFile.FileName = (String.IsNullOrEmpty(datFile.FileName) ? value : datFile.FileName);
break;
case "DatFile.Name":
datFile.Name = (String.IsNullOrEmpty(datFile.Name) ? value : datFile.Name);
break;
case "DatFile.Description":
datFile.Description = (String.IsNullOrEmpty(datFile.Description) ? value : datFile.Description);
break;
case "Machine.Name":
machineName = value;
break;
case "Description":
machineDesc = value;
break;
case "DatItem.Type":
switch (value.ToLowerInvariant())
{
case "archive":
itemType = ItemType.Archive;
break;
case "biosset":
itemType = ItemType.BiosSet;
break;
case "disk":
itemType = ItemType.Disk;
break;
case "release":
itemType = ItemType.Release;
break;
case "rom":
itemType = ItemType.Rom;
break;
case "sample":
itemType = ItemType.Sample;
break;
}
break;
case "Rom.Name":
case "Disk.Name":
name = value == "" ? name : value;
break;
case "DatItem.Size":
if (!Int64.TryParse(value, out size))
{
size = -1;
}
break;
case "DatItem.CRC":
crc = value;
break;
case "DatItem.MD5":
md5 = value;
break;
case "DatItem.SHA1":
sha1 = value;
break;
case "DatItem.SHA256":
sha256 = value;
break;
case "DatItem.SHA384":
sha384 = value;
break;
case "DatItem.SHA512":
sha512 = value;
break;
case "DatItem.Nodump":
switch (value.ToLowerInvariant())
{
case "baddump":
status = ItemStatus.BadDump;
break;
case "good":
status = ItemStatus.Good;
break;
case "no":
case "none":
status = ItemStatus.None;
break;
case "nodump":
case "yes":
status = ItemStatus.Nodump;
break;
case "verified":
status = ItemStatus.Verified;
break;
}
break;
}
}
// And now we populate and add the new item
switch (itemType)
{
case ItemType.Archive:
Archive archive = new Archive()
{
Name = name,
MachineName = machineName,
MachineDescription = machineDesc,
};
datFile.ParseAddHelper(archive, clean, remUnicode);
break;
case ItemType.BiosSet:
BiosSet biosset = new BiosSet()
{
Name = name,
MachineName = machineName,
Description = machineDesc,
};
datFile.ParseAddHelper(biosset, clean, remUnicode);
break;
case ItemType.Disk:
Disk disk = new Disk()
{
Name = name,
MD5 = md5,
SHA1 = sha1,
SHA256 = sha256,
SHA384 = sha384,
SHA512 = sha512,
MachineName = machineName,
MachineDescription = machineDesc,
ItemStatus = status,
};
datFile.ParseAddHelper(disk, clean, remUnicode);
break;
case ItemType.Release:
Release release = new Release()
{
Name = name,
MachineName = machineName,
MachineDescription = machineDesc,
};
datFile.ParseAddHelper(release, clean, remUnicode);
break;
case ItemType.Rom:
Rom rom = new Rom()
{
Name = name,
Size = size,
CRC = crc,
MD5 = md5,
SHA1 = sha1,
SHA256 = sha256,
SHA384 = sha384,
SHA512 = sha512,
MachineName = machineName,
MachineDescription = machineDesc,
ItemStatus = status,
};
datFile.ParseAddHelper(rom, clean, remUnicode);
break;
case ItemType.Sample:
Sample sample = new Sample()
{
Name = name,
MachineName = machineName,
MachineDescription = machineDesc,
};
datFile.ParseAddHelper(sample, clean, remUnicode);
break;
}
}
}
/// <summary>
/// Create and open an output file for writing direct from a dictionary
/// </summary>
/// <param name="datFile">DatFile to write out from</param>
/// <param name="outfile">Name of the file to write to</param>
/// <param name="delim">Delimiter for parsing individual lines</param>
/// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
/// <returns>True if the DAT was written correctly, false otherwise</returns>
public static bool WriteToFile(DatFile datFile, string outfile, char delim, bool ignoreblanks = false)
{
try
{
Globals.Logger.User("Opening file for writing: {0}", outfile);
FileStream fs = FileTools.TryCreate(outfile);
// If we get back null for some reason, just log and return
if (fs == null)
{
Globals.Logger.Warning("File '{0}' could not be created for writing! Please check to see if the file is writable", outfile);
return false;
}
StreamWriter sw = new StreamWriter(fs, new UTF8Encoding(true));
// Write out the header
WriteHeader(sw, delim);
// Get a properly sorted set of keys
List<string> keys = datFile.Keys.ToList();
keys.Sort(new NaturalComparer());
foreach (string key in keys)
{
List<DatItem> roms = datFile[key];
// Resolve the names in the block
roms = DatItem.ResolveNames(roms);
for (int index = 0; index < roms.Count; index++)
{
DatItem rom = roms[index];
// There are apparently times when a null rom can skip by, skip them
if (rom.Name == null || rom.MachineName == null)
{
Globals.Logger.Warning("Null rom found!");
continue;
}
// If we have a "null" game (created by DATFromDir or something similar), log it to file
if (rom.Type == ItemType.Rom
&& ((Rom)rom).Size == -1
&& ((Rom)rom).CRC == "null")
{
Globals.Logger.Verbose("Empty folder found: {0}", rom.MachineName);
}
// Now, output the rom data
WriteRomData(datFile, sw, delim, rom, ignoreblanks);
}
}
Globals.Logger.Verbose("File written!" + Environment.NewLine);
sw.Dispose();
fs.Dispose();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
/// <summary>
/// Write out DAT header using the supplied StreamWriter
/// </summary>
/// <param name="sw">StreamWriter to output to</param>
/// <param name="delim">Delimiter for parsing individual lines</param>
/// <returns>True if the data was written, false on error</returns>
private static bool WriteHeader(StreamWriter sw, char delim)
{
try
{
string header = string.Format("\"File Name\"{0}\"Internal Name\"{0}\"Description\"{0}\"Game Name\"{0}\"Game Description\"{0}\"Type\"{0}\"" +
"Rom Name\"{0}\"Disk Name\"{0}\"Size\"{0}\"CRC\"{0}\"MD5\"{0}\"SHA1\"{0}\"SHA256\"{0}\"Nodump\"\n", delim);
// Write the header out
sw.Write(header);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
/// <summary>
/// Write out RomData using the supplied StreamWriter
/// </summary>
/// <param name="datFile">DatFile to write out from</param>
/// <param name="sw">StreamWriter to output to</param>
/// <param name="delim">Delimiter for parsing individual lines</param>
/// <param name="rom">RomData object to be output</param>
/// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
/// <returns>True if the data was written, false on error</returns>
private static bool WriteRomData(DatFile datFile, StreamWriter sw, char delim, DatItem rom, bool ignoreblanks = false)
{
// If we are in ignore blanks mode AND we have a blank (0-size) rom, skip
if (ignoreblanks
&& (rom.Type == ItemType.Rom
&& (((Rom)rom).Size == 0 || ((Rom)rom).Size == -1)))
{
return true;
}
try
{
string state = "", name = "", pre = "", post = "";
// Separated values should only output Rom and Disk
if (rom.Type != ItemType.Disk && rom.Type != ItemType.Rom)
{
return true;
}
pre = datFile.Prefix + (datFile.Quotes ? "\"" : "");
post = (datFile.Quotes ? "\"" : "") + datFile.Postfix;
if (rom.Type == ItemType.Rom)
{
// Check for special strings in prefix and postfix
pre = pre
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", ((Rom)rom).CRC)
.Replace("%md5%", ((Rom)rom).MD5)
.Replace("%sha1%", ((Rom)rom).SHA1)
.Replace("%sha256%", ((Rom)rom).SHA256)
.Replace("%sha384%", ((Rom)rom).SHA384)
.Replace("%sha512%", ((Rom)rom).SHA512)
.Replace("%size%", ((Rom)rom).Size.ToString());
post = post
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", ((Rom)rom).CRC)
.Replace("%md5%", ((Rom)rom).MD5)
.Replace("%sha1%", ((Rom)rom).SHA1)
.Replace("%sha256%", ((Rom)rom).SHA256)
.Replace("%sha384%", ((Rom)rom).SHA384)
.Replace("%sha512%", ((Rom)rom).SHA512)
.Replace("%size%", ((Rom)rom).Size.ToString());
}
else if (rom.Type == ItemType.Disk)
{
// Check for special strings in prefix and postfix
pre = pre
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", string.Empty)
.Replace("%md5%", ((Disk)rom).MD5)
.Replace("%sha1%", ((Disk)rom).SHA1)
.Replace("%sha256%", ((Disk)rom).SHA256)
.Replace("%sha384%", ((Disk)rom).SHA384)
.Replace("%sha512%", ((Disk)rom).SHA512)
.Replace("%size%", string.Empty); ;
post = post
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", string.Empty)
.Replace("%md5%", ((Disk)rom).MD5)
.Replace("%sha1%", ((Disk)rom).SHA1)
.Replace("%sha256%", ((Disk)rom).SHA256)
.Replace("%sha384%", ((Disk)rom).SHA384)
.Replace("%sha512%", ((Disk)rom).SHA512)
.Replace("%size%", string.Empty); ;
}
else
{
// Check for special strings in prefix and postfix
pre = pre
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", string.Empty)
.Replace("%md5%", string.Empty)
.Replace("%sha1%", string.Empty)
.Replace("%size%", string.Empty);
post = post
.Replace("%game%", rom.MachineName)
.Replace("%name%", rom.Name)
.Replace("%crc%", string.Empty)
.Replace("%md5%", string.Empty)
.Replace("%sha1%", string.Empty)
.Replace("%size%", string.Empty);
}
if (rom.Type == ItemType.Rom)
{
string inline = string.Format("\"" + datFile.FileName + "\""
+ "{0}\"" + datFile.Name + "\""
+ "{0}\"" + datFile.Description + "\""
+ "{0}\"" + rom.MachineName + "\""
+ "{0}\"" + rom.MachineDescription + "\""
+ "{0}" + "\"rom\""
+ "{0}\"" + rom.Name + "\""
+ "{0}" + "\"\""
+ "{0}\"" + ((Rom)rom).Size + "\""
+ "{0}\"" + ((Rom)rom).CRC + "\""
+ "{0}\"" + ((Rom)rom).MD5 + "\""
+ "{0}\"" + ((Rom)rom).SHA1 + "\""
+ "{0}\"" + ((Rom)rom).SHA256 + "\""
// + "{0}\"" + ((Rom)rom).SHA384 + "\""
// + "{0}\"" + ((Rom)rom).SHA512 + "\""
+ "{0}" + (((Rom)rom).ItemStatus != ItemStatus.None ? "\"" + ((Rom)rom).ItemStatus.ToString() + "\"" : "\"\""), delim);
state += pre + inline + post + "\n";
}
else if (rom.Type == ItemType.Disk)
{
string inline = string.Format("\"" + datFile.FileName + "\""
+ "{0}\"" + datFile.Name + "\""
+ "{0}\"" + datFile.Description + "\""
+ "{0}\"" + rom.MachineName + "\""
+ "{0}\"" + rom.MachineDescription + "\""
+ "{0}" + "\"disk\""
+ "{0}" + "\"\""
+ "{0}\"" + rom.Name + "\""
+ "{0}" + "\"\""
+ "{0}" + "\"\""
+ "{0}\"" + ((Disk)rom).MD5 + "\""
+ "{0}\"" + ((Disk)rom).SHA1 + "\""
+ "{0}\"" + ((Disk)rom).SHA256 + "\""
// + "{0}\"" + ((Rom)rom).SHA384 + "\""
// + "{0}\"" + ((Rom)rom).SHA512 + "\""
+ "{0}" + (((Disk)rom).ItemStatus != ItemStatus.None ? "\"" + ((Disk)rom).ItemStatus.ToString() + "\"" : "\"\""), delim);
state += pre + inline + post + "\n";
}
sw.Write(state);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
}
}