using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Web;
using System.Xml;
using SabreTools.Library.Data;
using SabreTools.Library.DatItems;
using SabreTools.Library.Tools;
#if MONO
using System.IO;
#else
using Alphaleonis.Win32.Filesystem;
using FileStream = System.IO.FileStream;
using StreamWriter = System.IO.StreamWriter;
#endif
using NaturalSort;
namespace SabreTools.Library.DatFiles
{
///
/// Represents parsing and writing of an SabreDat XML DAT
///
/// TODO: Verify that all write for this DatFile type is correct
internal class SabreDat : DatFile
{
///
/// Constructor designed for casting a base DatFile
///
/// Parent DatFile to copy from
public SabreDat(DatFile datFile)
: base(datFile, cloneHeader: false)
{
}
///
/// Parse an SabreDat XML DAT and return all found directories and files within
///
/// Name of the file to be parsed
/// System ID for the DAT
/// Source ID for the DAT
/// True if full pathnames are to be kept, false otherwise (default)
/// True if game names are sanitized, false otherwise (default)
/// True if we should remove non-ASCII characters from output, false otherwise (default)
public override void ParseFile(
// Standard Dat parsing
string filename,
int sysid,
int srcid,
// Miscellaneous
bool keep,
bool clean,
bool remUnicode)
{
// Prepare all internal variables
bool empty = true;
string key = "";
List parent = new List();
Encoding enc = Utilities.GetEncoding(filename);
XmlReader xtr = Utilities.GetXmlTextReader(filename);
// If we got a null reader, just return
if (xtr == null)
{
return;
}
// Otherwise, read the file to the end
try
{
xtr.MoveToContent();
while (!xtr.EOF)
{
// If we're ending a folder or game, take care of possibly empty games and removing from the parent
if (xtr.NodeType == XmlNodeType.EndElement && (xtr.Name == "directory" || xtr.Name == "dir"))
{
// If we didn't find any items in the folder, make sure to add the blank rom
if (empty)
{
string tempgame = String.Join("\\", parent);
Rom rom = new Rom("null", tempgame, omitFromScan: Hash.DeepHashes); // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
// Now process and add the rom
key = ParseAddHelper(rom, clean, remUnicode);
}
// Regardless, end the current folder
int parentcount = parent.Count;
if (parentcount == 0)
{
Globals.Logger.Verbose("Empty parent '{0}' found in '{1}'", String.Join("\\", parent), filename);
empty = true;
}
// If we have an end folder element, remove one item from the parent, if possible
if (parentcount > 0)
{
parent.RemoveAt(parent.Count - 1);
if (keep && parentcount > 1)
{
Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type);
}
}
}
// We only want elements
if (xtr.NodeType != XmlNodeType.Element)
{
xtr.Read();
continue;
}
switch (xtr.Name)
{
// We want to process the entire subtree of the header
case "header":
ReadHeader(xtr.ReadSubtree(), keep);
// Skip the header node now that we've processed it
xtr.Skip();
break;
case "dir":
case "directory":
empty = ReadDirectory(xtr.ReadSubtree(), parent, filename, sysid, srcid, keep, clean, remUnicode);
// Skip the directory node now that we've processed it
xtr.Read();
break;
default:
xtr.Read();
break;
}
}
}
catch (Exception ex)
{
Globals.Logger.Warning("Exception found while parsing '{0}': {1}", filename, ex);
// For XML errors, just skip the affected node
xtr?.Read();
}
xtr.Dispose();
}
///
/// Read header information
///
/// XmlReader to use to parse the header
/// True if full pathnames are to be kept, false otherwise (default)
private void ReadHeader(XmlReader reader, bool keep)
{
bool superdat = false;
// If there's no subtree to the header, skip it
if (reader == null)
{
return;
}
// Otherwise, read what we can from the header
while (!reader.EOF)
{
// We only want elements
if (reader.NodeType != XmlNodeType.Element || reader.Name == "header")
{
reader.Read();
continue;
}
// Get all header items (ONLY OVERWRITE IF THERE'S NO DATA)
string content = "";
switch (reader.Name)
{
case "name":
content = reader.ReadElementContentAsString(); ;
Name = (String.IsNullOrWhiteSpace(Name) ? content : Name);
superdat = superdat || content.Contains(" - SuperDAT");
if (keep && superdat)
{
Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type);
}
break;
case "description":
content = reader.ReadElementContentAsString();
Description = (String.IsNullOrWhiteSpace(Description) ? content : Description);
break;
case "rootdir":
content = reader.ReadElementContentAsString();
RootDir = (String.IsNullOrWhiteSpace(RootDir) ? content : RootDir);
break;
case "category":
content = reader.ReadElementContentAsString();
Category = (String.IsNullOrWhiteSpace(Category) ? content : Category);
break;
case "version":
content = reader.ReadElementContentAsString();
Version = (String.IsNullOrWhiteSpace(Version) ? content : Version);
break;
case "date":
content = reader.ReadElementContentAsString();
Date = (String.IsNullOrWhiteSpace(Date) ? content.Replace(".", "/") : Date);
break;
case "author":
content = reader.ReadElementContentAsString();
Author = (String.IsNullOrWhiteSpace(Author) ? content : Author);
Email = (String.IsNullOrWhiteSpace(Email) ? reader.GetAttribute("email") : Email);
Homepage = (String.IsNullOrWhiteSpace(Homepage) ? reader.GetAttribute("homepage") : Homepage);
Url = (String.IsNullOrWhiteSpace(Url) ? reader.GetAttribute("url") : Url);
break;
case "comment":
content = reader.ReadElementContentAsString();
Comment = (String.IsNullOrWhiteSpace(Comment) ? content : Comment);
break;
case "flags":
ReadFlags(reader.ReadSubtree(), superdat);
// Skip the flags node now that we've processed it
reader.Skip();
break;
default:
reader.Read();
break;
}
}
}
///
/// Read directory information
///
/// XmlReader to use to parse the header
/// Name of the file to be parsed
/// System ID for the DAT
/// Source ID for the DAT
/// True if full pathnames are to be kept, false otherwise (default)
/// True if game names are sanitized, false otherwise (default)
/// True if we should remove non-ASCII characters from output, false otherwise (default)
private bool ReadDirectory(XmlReader reader,
List parent,
// Standard Dat parsing
string filename,
int sysid,
int srcid,
// Miscellaneous
bool keep,
bool clean,
bool remUnicode)
{
// Prepare all internal variables
XmlReader flagreader;
bool empty = true;
string key = "", date = "";
long size = -1;
ItemStatus its = ItemStatus.None;
// If there's no subtree to the header, skip it
if (reader == null)
{
return empty;
}
string foldername = (reader.GetAttribute("name") ?? "");
if (!String.IsNullOrWhiteSpace(foldername))
{
parent.Add(foldername);
}
// Otherwise, read what we can from the directory
while (!reader.EOF)
{
// If we're ending a folder or game, take care of possibly empty games and removing from the parent
if (reader.NodeType == XmlNodeType.EndElement && (reader.Name == "directory" || reader.Name == "dir"))
{
// If we didn't find any items in the folder, make sure to add the blank rom
if (empty)
{
string tempgame = String.Join("\\", parent);
Rom rom = new Rom("null", tempgame, omitFromScan: Hash.DeepHashes); // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
// Now process and add the rom
key = ParseAddHelper(rom, clean, remUnicode);
}
// Regardless, end the current folder
int parentcount = parent.Count;
if (parentcount == 0)
{
Globals.Logger.Verbose("Empty parent '{0}' found in '{1}'", String.Join("\\", parent), filename);
empty = true;
}
// If we have an end folder element, remove one item from the parent, if possible
if (parentcount > 0)
{
parent.RemoveAt(parent.Count - 1);
if (keep && parentcount > 1)
{
Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type);
}
}
}
// We only want elements
if (reader.NodeType != XmlNodeType.Element)
{
reader.Read();
continue;
}
// Get all directory items
string content = "";
switch (reader.Name)
{
// Directories can contain directories
case "dir":
case "directory":
ReadDirectory(reader.ReadSubtree(), parent, filename, sysid, srcid, keep, clean, remUnicode);
// Skip the directory node now that we've processed it
reader.Read();
break;
case "file":
empty = false;
// If the rom is itemStatus, flag it
its = ItemStatus.None;
flagreader = reader.ReadSubtree();
// If the subtree is empty, skip it
if (flagreader == null)
{
reader.Skip();
continue;
}
while (!flagreader.EOF)
{
// We only want elements
if (flagreader.NodeType != XmlNodeType.Element || flagreader.Name == "flags")
{
flagreader.Read();
continue;
}
switch (flagreader.Name)
{
case "flag":
if (flagreader.GetAttribute("name") != null && flagreader.GetAttribute("value") != null)
{
content = flagreader.GetAttribute("value");
its = Utilities.GetItemStatus(flagreader.GetAttribute("name"));
}
break;
}
flagreader.Read();
}
// If the rom has a Date attached, read it in and then sanitize it
date = Utilities.GetDate(reader.GetAttribute("date"));
// Take care of hex-sized files
size = Utilities.GetSize(reader.GetAttribute("size"));
Machine dir = new Machine();
// Get the name of the game from the parent
dir.Name = String.Join("\\", parent);
dir.Description = dir.Name;
DatItem datItem;
switch (reader.GetAttribute("type").ToLowerInvariant())
{
case "archive":
datItem = new Archive
{
Name = reader.GetAttribute("name"),
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
case "biosset":
datItem = new BiosSet
{
Name = reader.GetAttribute("name"),
Description = reader.GetAttribute("description"),
Default = Utilities.GetYesNo(reader.GetAttribute("default")),
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
case "disk":
datItem = new Disk
{
Name = reader.GetAttribute("name"),
MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length),
SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length),
SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length),
SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length),
SHA512 = Utilities.CleanHashData(reader.GetAttribute("sha512"), Constants.SHA512Length),
ItemStatus = its,
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
case "release":
datItem = new Release
{
Name = reader.GetAttribute("name"),
Region = reader.GetAttribute("region"),
Language = reader.GetAttribute("language"),
Date = reader.GetAttribute("date"),
Default = Utilities.GetYesNo(reader.GetAttribute("default")),
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
case "rom":
datItem = new Rom
{
Name = reader.GetAttribute("name"),
Size = size,
CRC = Utilities.CleanHashData(reader.GetAttribute("crc"), Constants.CRCLength),
MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length),
SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length),
SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length),
SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length),
SHA512 = Utilities.CleanHashData(reader.GetAttribute("sha512"), Constants.SHA512Length),
ItemStatus = its,
Date = date,
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
case "sample":
datItem = new Sample
{
Name = reader.GetAttribute("name"),
SystemID = sysid,
System = filename,
SourceID = srcid,
};
break;
default:
// By default, create a new Blank, just in case
datItem = new Blank();
break;
}
datItem?.CopyMachineInformation(dir);
// Now process and add the rom
key = ParseAddHelper(datItem, clean, remUnicode);
reader.Read();
break;
}
}
return empty;
}
///
/// Read flags information
///
/// XmlReader to use to parse the header
/// True if superdat has already been set externally, false otherwise
private void ReadFlags(XmlReader reader, bool superdat)
{
// Prepare all internal variables
string content = "";
// If we somehow have a null flag section, skip it
if (reader == null)
{
return;
}
while (!reader.EOF)
{
// We only want elements
if (reader.NodeType != XmlNodeType.Element || reader.Name == "flags")
{
reader.Read();
continue;
}
switch (reader.Name)
{
case "flag":
if (reader.GetAttribute("name") != null && reader.GetAttribute("value") != null)
{
content = reader.GetAttribute("value");
switch (reader.GetAttribute("name").ToLowerInvariant())
{
case "type":
Type = (String.IsNullOrWhiteSpace(Type) ? content : Type);
superdat = superdat || content.Contains("SuperDAT");
break;
case "forcemerging":
if (ForceMerging == ForceMerging.None)
{
ForceMerging = Utilities.GetForceMerging(content);
}
break;
case "forcenodump":
if (ForceNodump == ForceNodump.None)
{
ForceNodump = Utilities.GetForceNodump(content);
}
break;
case "forcepacking":
if (ForcePacking == ForcePacking.None)
{
ForcePacking = Utilities.GetForcePacking(content);
}
break;
}
}
reader.Read();
break;
default:
reader.Read();
break;
}
}
}
///
/// Create and open an output file for writing direct from a dictionary
///
/// Name of the file to write to
/// True if blank roms should be skipped on output, false otherwise (default)
/// True if the DAT was written correctly, false otherwise
/// TODO: Fix writing out files that have a path in the name
public override bool WriteToFile(string outfile, bool ignoreblanks = false)
{
try
{
Globals.Logger.User("Opening file for writing: {0}", outfile);
FileStream fs = Utilities.TryCreate(outfile);
// If we get back null for some reason, just log and return
if (fs == null)
{
Globals.Logger.Warning("File '{0}' could not be created for writing! Please check to see if the file is writable", outfile);
return false;
}
StreamWriter sw = new StreamWriter(fs, new UTF8Encoding(false));
// Write out the header
WriteHeader(sw);
// Write out each of the machines and roms
int depth = 2, last = -1;
string lastgame = null;
List splitpath = new List();
// Get a properly sorted set of keys
List keys = Keys;
keys.Sort(new NaturalComparer());
foreach (string key in keys)
{
List roms = this[key];
// Resolve the names in the block
roms = DatItem.ResolveNames(roms);
for (int index = 0; index < roms.Count; index++)
{
DatItem rom = roms[index];
// There are apparently times when a null rom can skip by, skip them
if (rom.Name == null || rom.MachineName == null)
{
Globals.Logger.Warning("Null rom found!");
continue;
}
List newsplit = rom.MachineName.Split('\\').ToList();
// If we have a different game and we're not at the start of the list, output the end of last item
if (lastgame != null && lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant())
{
depth = WriteEndGame(sw, splitpath, newsplit, depth, out last);
}
// If we have a new game, output the beginning of the new item
if (lastgame == null || lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant())
{
depth = WriteStartGame(sw, rom, newsplit, lastgame, depth, last);
}
// If we have a "null" game (created by DATFromDir or something similar), log it to file
if (rom.ItemType == ItemType.Rom
&& ((Rom)rom).Size == -1
&& ((Rom)rom).CRC == "null")
{
Globals.Logger.Verbose("Empty folder found: {0}", rom.MachineName);
splitpath = newsplit;
lastgame = rom.MachineName;
continue;
}
// Now, output the rom data
WriteDatItem(sw, rom, depth, ignoreblanks);
// Set the new data to compare against
splitpath = newsplit;
lastgame = rom.MachineName;
}
}
// Write the file footer out
WriteFooter(sw, depth);
Globals.Logger.Verbose("File written!" + Environment.NewLine);
sw.Dispose();
fs.Dispose();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
///
/// Write out DAT header using the supplied StreamWriter
///
/// StreamWriter to output to
/// True if the data was written, false on error
private bool WriteHeader(StreamWriter sw)
{
try
{
string header = "\n" +
"\n\n" +
"\n" +
"\t\n" +
"\t\t" + HttpUtility.HtmlEncode(Name) + "\n" +
"\t\t" + HttpUtility.HtmlEncode(Description) + "\n" +
(!String.IsNullOrWhiteSpace(RootDir) ? "\t\t" + HttpUtility.HtmlEncode(RootDir) + "\n" : "") +
(!String.IsNullOrWhiteSpace(Category) ? "\t\t" + HttpUtility.HtmlEncode(Category) + "\n" : "") +
"\t\t" + HttpUtility.HtmlEncode(Version) + "\n" +
(!String.IsNullOrWhiteSpace(Date) ? "\t\t" + HttpUtility.HtmlEncode(Date) + "\n" : "") +
"\t\t" + HttpUtility.HtmlEncode(Author) + "\n" +
(!String.IsNullOrWhiteSpace(Comment) ? "\t\t" + HttpUtility.HtmlEncode(Comment) + "\n" : "") +
(!String.IsNullOrWhiteSpace(Type) || ForcePacking != ForcePacking.None || ForceMerging != ForceMerging.None || ForceNodump != ForceNodump.None ?
"\t\t\n" +
(!String.IsNullOrWhiteSpace(Type) ? "\t\t\t\n" : "") +
(ForcePacking == ForcePacking.Unzip ? "\t\t\t\n" : "") +
(ForcePacking == ForcePacking.Zip ? "\t\t\t\n" : "") +
(ForceMerging == ForceMerging.Full ? "\t\t\t\n" : "") +
(ForceMerging == ForceMerging.Split ? "\t\t\t\n" : "") +
(ForceMerging == ForceMerging.Merged ? "\t\t\t\n" : "") +
(ForceMerging == ForceMerging.NonMerged ? "\t\t\t\n" : "") +
(ForceNodump == ForceNodump.Ignore ? "\t\t\t\n" : "") +
(ForceNodump == ForceNodump.Obsolete ? "\t\t\t\n" : "") +
(ForceNodump == ForceNodump.Required ? "\t\t\t\n" : "") +
"\t\t\n"
: "") +
"\t\n" +
"\t\n";
// Write the header out
sw.Write(header);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
///
/// Write out Game start using the supplied StreamWriter
///
/// StreamWriter to output to
/// DatItem object to be output
/// Split path representing the parent game (SabreDAT only)
/// The name of the last game to be output
/// Current depth to output file at (SabreDAT only)
/// Last known depth to cycle back from (SabreDAT only)
/// The new depth of the tag
private int WriteStartGame(StreamWriter sw, DatItem rom, List newsplit, string lastgame, int depth, int last)
{
try
{
// No game should start with a path separator
if (rom.MachineName.StartsWith(Path.DirectorySeparatorChar.ToString()))
{
rom.MachineName = rom.MachineName.Substring(1);
}
string state = "";
for (int i = (last == -1 ? 0 : last); i < newsplit.Count; i++)
{
for (int j = 0; j < depth - last + i - (lastgame == null ? 1 : 0); j++)
{
state += "\t";
}
state += "\n";
}
depth = depth - (last == -1 ? 0 : last) + newsplit.Count;
sw.Write(state);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return depth;
}
return depth;
}
///
/// Write out Game start using the supplied StreamWriter
///
/// StreamWriter to output to
/// Split path representing last kwown parent game (SabreDAT only)
/// Split path representing the parent game (SabreDAT only)
/// Current depth to output file at (SabreDAT only)
/// Last known depth to cycle back from (SabreDAT only)
/// The new depth of the tag
private int WriteEndGame(StreamWriter sw, List splitpath, List newsplit, int depth, out int last)
{
last = 0;
try
{
string state = "";
if (splitpath != null)
{
for (int i = 0; i < newsplit.Count && i < splitpath.Count; i++)
{
// Always keep track of the last seen item
last = i;
// If we find a difference, break
if (newsplit[i] != splitpath[i])
{
break;
}
}
// Now that we have the last known position, take down all open folders
for (int i = depth - 1; i > last + 1; i--)
{
// Print out the number of tabs and the end folder
for (int j = 0; j < i; j++)
{
state += "\t";
}
state += "\n";
}
// Reset the current depth
depth = 2 + last;
}
sw.Write(state);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return depth;
}
return depth;
}
///
/// Write out DatItem using the supplied StreamWriter
///
/// StreamWriter to output to
/// DatItem object to be output
/// Current depth to output file at (SabreDAT only)
/// True if blank roms should be skipped on output, false otherwise (default)
/// True if the data was written, false on error
private bool WriteDatItem(StreamWriter sw, DatItem rom, int depth, bool ignoreblanks = false)
{
// If we are in ignore blanks mode AND we have a blank (0-size) rom, skip
if (ignoreblanks
&& (rom.ItemType == ItemType.Rom
&& (((Rom)rom).Size == 0 || ((Rom)rom).Size == -1)))
{
return true;
}
try
{
string state = "", prefix = "";
// Pre-process the item name
ProcessItemName(rom, true);
for (int i = 0; i < depth; i++)
{
prefix += "\t";
}
state += prefix;
switch (rom.ItemType)
{
case ItemType.Archive:
state += "\n";
break;
case ItemType.BiosSet:
state += "\n";
break;
case ItemType.Disk:
state += "\n" + prefix + "\t\n" +
prefix + "\t\t\n" +
prefix + "\t\n" +
prefix + "\n" : "/>\n");
break;
case ItemType.Release:
state += "\n";
break;
case ItemType.Rom:
state += "\n" + prefix + "\t\n" +
prefix + "\t\t\n" +
prefix + "\t\n" +
prefix + "\n" : "/>\n");
break;
case ItemType.Sample:
state += "\n";
break;
}
sw.Write(state);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
///
/// Write out DAT footer using the supplied StreamWriter
///
/// StreamWriter to output to
/// Current depth to output file at (SabreDAT only)
/// True if the data was written, false on error
private bool WriteFooter(StreamWriter sw, int depth)
{
try
{
string footer = "";
for (int i = depth - 1; i >= 2; i--)
{
// Print out the number of tabs and the end folder
for (int j = 0; j < i; j++)
{
footer += "\t";
}
footer += "\n";
}
footer += "\t\n\n";
// Write the footer out
sw.Write(footer);
sw.Flush();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return false;
}
return true;
}
}
}