using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Web; using System.Xml; using SabreTools.Library.Data; using SabreTools.Library.DatItems; using SabreTools.Library.Tools; #if MONO using System.IO; #else using Alphaleonis.Win32.Filesystem; using FileStream = System.IO.FileStream; using StreamWriter = System.IO.StreamWriter; #endif using NaturalSort; namespace SabreTools.Library.DatFiles { /// /// Represents parsing and writing of an SabreDat XML DAT /// /// TODO: Verify that all write for this DatFile type is correct internal class SabreDat : DatFile { /// /// Constructor designed for casting a base DatFile /// /// Parent DatFile to copy from public SabreDat(DatFile datFile) : base(datFile, cloneHeader: false) { } /// /// Parse an SabreDat XML DAT and return all found directories and files within /// /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) public override void ParseFile( // Standard Dat parsing string filename, int sysid, int srcid, // Miscellaneous bool keep, bool clean, bool remUnicode) { // Prepare all internal variables bool empty = true; string key = ""; List parent = new List(); Encoding enc = Utilities.GetEncoding(filename); XmlReader xtr = Utilities.GetXmlTextReader(filename); // If we got a null reader, just return if (xtr == null) { return; } // Otherwise, read the file to the end try { xtr.MoveToContent(); while (!xtr.EOF) { // If we're ending a folder or game, take care of possibly empty games and removing from the parent if (xtr.NodeType == XmlNodeType.EndElement && (xtr.Name == "directory" || xtr.Name == "dir")) { // If we didn't find any items in the folder, make sure to add the blank rom if (empty) { string tempgame = String.Join("\\", parent); Rom rom = new Rom("null", tempgame, omitFromScan: Hash.DeepHashes); // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually // Now process and add the rom key = ParseAddHelper(rom, clean, remUnicode); } // Regardless, end the current folder int parentcount = parent.Count; if (parentcount == 0) { Globals.Logger.Verbose("Empty parent '{0}' found in '{1}'", String.Join("\\", parent), filename); empty = true; } // If we have an end folder element, remove one item from the parent, if possible if (parentcount > 0) { parent.RemoveAt(parent.Count - 1); if (keep && parentcount > 1) { Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type); } } } // We only want elements if (xtr.NodeType != XmlNodeType.Element) { xtr.Read(); continue; } switch (xtr.Name) { // We want to process the entire subtree of the header case "header": ReadHeader(xtr.ReadSubtree(), keep); // Skip the header node now that we've processed it xtr.Skip(); break; case "dir": case "directory": empty = ReadDirectory(xtr.ReadSubtree(), parent, filename, sysid, srcid, keep, clean, remUnicode); // Skip the directory node now that we've processed it xtr.Read(); break; default: xtr.Read(); break; } } } catch (Exception ex) { Globals.Logger.Warning("Exception found while parsing '{0}': {1}", filename, ex); // For XML errors, just skip the affected node xtr?.Read(); } xtr.Dispose(); } /// /// Read header information /// /// XmlReader to use to parse the header /// True if full pathnames are to be kept, false otherwise (default) private void ReadHeader(XmlReader reader, bool keep) { bool superdat = false; // If there's no subtree to the header, skip it if (reader == null) { return; } // Otherwise, read what we can from the header while (!reader.EOF) { // We only want elements if (reader.NodeType != XmlNodeType.Element || reader.Name == "header") { reader.Read(); continue; } // Get all header items (ONLY OVERWRITE IF THERE'S NO DATA) string content = ""; switch (reader.Name) { case "name": content = reader.ReadElementContentAsString(); ; Name = (String.IsNullOrWhiteSpace(Name) ? content : Name); superdat = superdat || content.Contains(" - SuperDAT"); if (keep && superdat) { Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type); } break; case "description": content = reader.ReadElementContentAsString(); Description = (String.IsNullOrWhiteSpace(Description) ? content : Description); break; case "rootdir": content = reader.ReadElementContentAsString(); RootDir = (String.IsNullOrWhiteSpace(RootDir) ? content : RootDir); break; case "category": content = reader.ReadElementContentAsString(); Category = (String.IsNullOrWhiteSpace(Category) ? content : Category); break; case "version": content = reader.ReadElementContentAsString(); Version = (String.IsNullOrWhiteSpace(Version) ? content : Version); break; case "date": content = reader.ReadElementContentAsString(); Date = (String.IsNullOrWhiteSpace(Date) ? content.Replace(".", "/") : Date); break; case "author": content = reader.ReadElementContentAsString(); Author = (String.IsNullOrWhiteSpace(Author) ? content : Author); Email = (String.IsNullOrWhiteSpace(Email) ? reader.GetAttribute("email") : Email); Homepage = (String.IsNullOrWhiteSpace(Homepage) ? reader.GetAttribute("homepage") : Homepage); Url = (String.IsNullOrWhiteSpace(Url) ? reader.GetAttribute("url") : Url); break; case "comment": content = reader.ReadElementContentAsString(); Comment = (String.IsNullOrWhiteSpace(Comment) ? content : Comment); break; case "flags": ReadFlags(reader.ReadSubtree(), superdat); // Skip the flags node now that we've processed it reader.Skip(); break; default: reader.Read(); break; } } } /// /// Read directory information /// /// XmlReader to use to parse the header /// Name of the file to be parsed /// System ID for the DAT /// Source ID for the DAT /// True if full pathnames are to be kept, false otherwise (default) /// True if game names are sanitized, false otherwise (default) /// True if we should remove non-ASCII characters from output, false otherwise (default) private bool ReadDirectory(XmlReader reader, List parent, // Standard Dat parsing string filename, int sysid, int srcid, // Miscellaneous bool keep, bool clean, bool remUnicode) { // Prepare all internal variables XmlReader flagreader; bool empty = true; string key = "", date = ""; long size = -1; ItemStatus its = ItemStatus.None; // If there's no subtree to the header, skip it if (reader == null) { return empty; } string foldername = (reader.GetAttribute("name") ?? ""); if (!String.IsNullOrWhiteSpace(foldername)) { parent.Add(foldername); } // Otherwise, read what we can from the directory while (!reader.EOF) { // If we're ending a folder or game, take care of possibly empty games and removing from the parent if (reader.NodeType == XmlNodeType.EndElement && (reader.Name == "directory" || reader.Name == "dir")) { // If we didn't find any items in the folder, make sure to add the blank rom if (empty) { string tempgame = String.Join("\\", parent); Rom rom = new Rom("null", tempgame, omitFromScan: Hash.DeepHashes); // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually // Now process and add the rom key = ParseAddHelper(rom, clean, remUnicode); } // Regardless, end the current folder int parentcount = parent.Count; if (parentcount == 0) { Globals.Logger.Verbose("Empty parent '{0}' found in '{1}'", String.Join("\\", parent), filename); empty = true; } // If we have an end folder element, remove one item from the parent, if possible if (parentcount > 0) { parent.RemoveAt(parent.Count - 1); if (keep && parentcount > 1) { Type = (String.IsNullOrWhiteSpace(Type) ? "SuperDAT" : Type); } } } // We only want elements if (reader.NodeType != XmlNodeType.Element) { reader.Read(); continue; } // Get all directory items string content = ""; switch (reader.Name) { // Directories can contain directories case "dir": case "directory": ReadDirectory(reader.ReadSubtree(), parent, filename, sysid, srcid, keep, clean, remUnicode); // Skip the directory node now that we've processed it reader.Read(); break; case "file": empty = false; // If the rom is itemStatus, flag it its = ItemStatus.None; flagreader = reader.ReadSubtree(); // If the subtree is empty, skip it if (flagreader == null) { reader.Skip(); continue; } while (!flagreader.EOF) { // We only want elements if (flagreader.NodeType != XmlNodeType.Element || flagreader.Name == "flags") { flagreader.Read(); continue; } switch (flagreader.Name) { case "flag": if (flagreader.GetAttribute("name") != null && flagreader.GetAttribute("value") != null) { content = flagreader.GetAttribute("value"); its = Utilities.GetItemStatus(flagreader.GetAttribute("name")); } break; } flagreader.Read(); } // If the rom has a Date attached, read it in and then sanitize it date = Utilities.GetDate(reader.GetAttribute("date")); // Take care of hex-sized files size = Utilities.GetSize(reader.GetAttribute("size")); Machine dir = new Machine(); // Get the name of the game from the parent dir.Name = String.Join("\\", parent); dir.Description = dir.Name; DatItem datItem; switch (reader.GetAttribute("type").ToLowerInvariant()) { case "archive": datItem = new Archive { Name = reader.GetAttribute("name"), SystemID = sysid, System = filename, SourceID = srcid, }; break; case "biosset": datItem = new BiosSet { Name = reader.GetAttribute("name"), Description = reader.GetAttribute("description"), Default = Utilities.GetYesNo(reader.GetAttribute("default")), SystemID = sysid, System = filename, SourceID = srcid, }; break; case "disk": datItem = new Disk { Name = reader.GetAttribute("name"), MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length), SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length), SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length), SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length), SHA512 = Utilities.CleanHashData(reader.GetAttribute("sha512"), Constants.SHA512Length), ItemStatus = its, SystemID = sysid, System = filename, SourceID = srcid, }; break; case "release": datItem = new Release { Name = reader.GetAttribute("name"), Region = reader.GetAttribute("region"), Language = reader.GetAttribute("language"), Date = reader.GetAttribute("date"), Default = Utilities.GetYesNo(reader.GetAttribute("default")), SystemID = sysid, System = filename, SourceID = srcid, }; break; case "rom": datItem = new Rom { Name = reader.GetAttribute("name"), Size = size, CRC = Utilities.CleanHashData(reader.GetAttribute("crc"), Constants.CRCLength), MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length), SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length), SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length), SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length), SHA512 = Utilities.CleanHashData(reader.GetAttribute("sha512"), Constants.SHA512Length), ItemStatus = its, Date = date, SystemID = sysid, System = filename, SourceID = srcid, }; break; case "sample": datItem = new Sample { Name = reader.GetAttribute("name"), SystemID = sysid, System = filename, SourceID = srcid, }; break; default: // By default, create a new Blank, just in case if (this.KeepEmptyGames) { datItem = new Blank(); } else { datItem = null; } break; } datItem?.CopyMachineInformation(dir); // Now process and add the rom key = ParseAddHelper(datItem, clean, remUnicode); reader.Read(); break; } } return empty; } /// /// Read flags information /// /// XmlReader to use to parse the header /// True if superdat has already been set externally, false otherwise private void ReadFlags(XmlReader reader, bool superdat) { // Prepare all internal variables string content = ""; // If we somehow have a null flag section, skip it if (reader == null) { return; } while (!reader.EOF) { // We only want elements if (reader.NodeType != XmlNodeType.Element || reader.Name == "flags") { reader.Read(); continue; } switch (reader.Name) { case "flag": if (reader.GetAttribute("name") != null && reader.GetAttribute("value") != null) { content = reader.GetAttribute("value"); switch (reader.GetAttribute("name").ToLowerInvariant()) { case "type": Type = (String.IsNullOrWhiteSpace(Type) ? content : Type); superdat = superdat || content.Contains("SuperDAT"); break; case "forcemerging": if (ForceMerging == ForceMerging.None) { ForceMerging = Utilities.GetForceMerging(content); } break; case "forcenodump": if (ForceNodump == ForceNodump.None) { ForceNodump = Utilities.GetForceNodump(content); } break; case "forcepacking": if (ForcePacking == ForcePacking.None) { ForcePacking = Utilities.GetForcePacking(content); } break; } } reader.Read(); break; default: reader.Read(); break; } } } /// /// Create and open an output file for writing direct from a dictionary /// /// Name of the file to write to /// True if blank roms should be skipped on output, false otherwise (default) /// True if the DAT was written correctly, false otherwise /// TODO: Fix writing out files that have a path in the name public override bool WriteToFile(string outfile, bool ignoreblanks = false) { try { Globals.Logger.User("Opening file for writing: {0}", outfile); FileStream fs = Utilities.TryCreate(outfile); // If we get back null for some reason, just log and return if (fs == null) { Globals.Logger.Warning("File '{0}' could not be created for writing! Please check to see if the file is writable", outfile); return false; } StreamWriter sw = new StreamWriter(fs, new UTF8Encoding(false)); // Write out the header WriteHeader(sw); // Write out each of the machines and roms int depth = 2, last = -1; string lastgame = null; List splitpath = new List(); // Get a properly sorted set of keys List keys = Keys; keys.Sort(new NaturalComparer()); foreach (string key in keys) { List roms = this[key]; // Resolve the names in the block roms = DatItem.ResolveNames(roms); for (int index = 0; index < roms.Count; index++) { DatItem rom = roms[index]; // There are apparently times when a null rom can skip by, skip them if (rom.Name == null || rom.MachineName == null) { Globals.Logger.Warning("Null rom found!"); continue; } List newsplit = rom.MachineName.Split('\\').ToList(); // If we have a different game and we're not at the start of the list, output the end of last item if (lastgame != null && lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant()) { depth = WriteEndGame(sw, splitpath, newsplit, depth, out last); } // If we have a new game, output the beginning of the new item if (lastgame == null || lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant()) { depth = WriteStartGame(sw, rom, newsplit, lastgame, depth, last); } // If we have a "null" game (created by DATFromDir or something similar), log it to file if (rom.Type == ItemType.Rom && ((Rom)rom).Size == -1 && ((Rom)rom).CRC == "null") { Globals.Logger.Verbose("Empty folder found: {0}", rom.MachineName); splitpath = newsplit; lastgame = rom.MachineName; continue; } // Now, output the rom data WriteDatItem(sw, rom, depth, ignoreblanks); // Set the new data to compare against splitpath = newsplit; lastgame = rom.MachineName; } } // Write the file footer out WriteFooter(sw, depth); Globals.Logger.Verbose("File written!" + Environment.NewLine); sw.Dispose(); fs.Dispose(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return false; } return true; } /// /// Write out DAT header using the supplied StreamWriter /// /// StreamWriter to output to /// True if the data was written, false on error private bool WriteHeader(StreamWriter sw) { try { string header = "\n" + "\n\n" + "\n" + "\t
\n" + "\t\t" + HttpUtility.HtmlEncode(Name) + "\n" + "\t\t" + HttpUtility.HtmlEncode(Description) + "\n" + (!String.IsNullOrWhiteSpace(RootDir) ? "\t\t" + HttpUtility.HtmlEncode(RootDir) + "\n" : "") + (!String.IsNullOrWhiteSpace(Category) ? "\t\t" + HttpUtility.HtmlEncode(Category) + "\n" : "") + "\t\t" + HttpUtility.HtmlEncode(Version) + "\n" + (!String.IsNullOrWhiteSpace(Date) ? "\t\t" + HttpUtility.HtmlEncode(Date) + "\n" : "") + "\t\t" + HttpUtility.HtmlEncode(Author) + "\n" + (!String.IsNullOrWhiteSpace(Comment) ? "\t\t" + HttpUtility.HtmlEncode(Comment) + "\n" : "") + (!String.IsNullOrWhiteSpace(Type) || ForcePacking != ForcePacking.None || ForceMerging != ForceMerging.None || ForceNodump != ForceNodump.None ? "\t\t\n" + (!String.IsNullOrWhiteSpace(Type) ? "\t\t\t\n" : "") + (ForcePacking == ForcePacking.Unzip ? "\t\t\t\n" : "") + (ForcePacking == ForcePacking.Zip ? "\t\t\t\n" : "") + (ForceMerging == ForceMerging.Full ? "\t\t\t\n" : "") + (ForceMerging == ForceMerging.Split ? "\t\t\t\n" : "") + (ForceMerging == ForceMerging.Merged ? "\t\t\t\n" : "") + (ForceMerging == ForceMerging.NonMerged ? "\t\t\t\n" : "") + (ForceNodump == ForceNodump.Ignore ? "\t\t\t\n" : "") + (ForceNodump == ForceNodump.Obsolete ? "\t\t\t\n" : "") + (ForceNodump == ForceNodump.Required ? "\t\t\t\n" : "") + "\t\t\n" : "") + "\t
\n" + "\t\n"; // Write the header out sw.Write(header); sw.Flush(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return false; } return true; } /// /// Write out Game start using the supplied StreamWriter /// /// StreamWriter to output to /// DatItem object to be output /// Split path representing the parent game (SabreDAT only) /// The name of the last game to be output /// Current depth to output file at (SabreDAT only) /// Last known depth to cycle back from (SabreDAT only) /// The new depth of the tag private int WriteStartGame(StreamWriter sw, DatItem rom, List newsplit, string lastgame, int depth, int last) { try { // No game should start with a path separator if (rom.MachineName.StartsWith(Path.DirectorySeparatorChar.ToString())) { rom.MachineName = rom.MachineName.Substring(1); } string state = ""; for (int i = (last == -1 ? 0 : last); i < newsplit.Count; i++) { for (int j = 0; j < depth - last + i - (lastgame == null ? 1 : 0); j++) { state += "\t"; } state += "\n"; } depth = depth - (last == -1 ? 0 : last) + newsplit.Count; sw.Write(state); sw.Flush(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return depth; } return depth; } /// /// Write out Game start using the supplied StreamWriter /// /// StreamWriter to output to /// Split path representing last kwown parent game (SabreDAT only) /// Split path representing the parent game (SabreDAT only) /// Current depth to output file at (SabreDAT only) /// Last known depth to cycle back from (SabreDAT only) /// The new depth of the tag private int WriteEndGame(StreamWriter sw, List splitpath, List newsplit, int depth, out int last) { last = 0; try { string state = ""; if (splitpath != null) { for (int i = 0; i < newsplit.Count && i < splitpath.Count; i++) { // Always keep track of the last seen item last = i; // If we find a difference, break if (newsplit[i] != splitpath[i]) { break; } } // Now that we have the last known position, take down all open folders for (int i = depth - 1; i > last + 1; i--) { // Print out the number of tabs and the end folder for (int j = 0; j < i; j++) { state += "\t"; } state += "\n"; } // Reset the current depth depth = 2 + last; } sw.Write(state); sw.Flush(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return depth; } return depth; } /// /// Write out DatItem using the supplied StreamWriter /// /// StreamWriter to output to /// DatItem object to be output /// Current depth to output file at (SabreDAT only) /// True if blank roms should be skipped on output, false otherwise (default) /// True if the data was written, false on error private bool WriteDatItem(StreamWriter sw, DatItem rom, int depth, bool ignoreblanks = false) { // If we are in ignore blanks mode AND we have a blank (0-size) rom, skip if (ignoreblanks && (rom.Type == ItemType.Rom && (((Rom)rom).Size == 0 || ((Rom)rom).Size == -1))) { return true; } try { string state = "", prefix = ""; // Pre-process the item name ProcessItemName(rom, true); for (int i = 0; i < depth; i++) { prefix += "\t"; } state += prefix; switch (rom.Type) { case ItemType.Archive: state += "\n"; break; case ItemType.BiosSet: state += "\n"; break; case ItemType.Disk: state += "\n" + prefix + "\t\n" + prefix + "\t\t\n" + prefix + "\t\n" + prefix + "\n" : "/>\n"); break; case ItemType.Release: state += "\n"; break; case ItemType.Rom: state += "\n" + prefix + "\t\n" + prefix + "\t\t\n" + prefix + "\t\n" + prefix + "\n" : "/>\n"); break; case ItemType.Sample: state += "\n"; break; } sw.Write(state); sw.Flush(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return false; } return true; } /// /// Write out DAT footer using the supplied StreamWriter /// /// StreamWriter to output to /// Current depth to output file at (SabreDAT only) /// True if the data was written, false on error private bool WriteFooter(StreamWriter sw, int depth) { try { string footer = ""; for (int i = depth - 1; i >= 2; i--) { // Print out the number of tabs and the end folder for (int j = 0; j < i; j++) { footer += "\t"; } footer += "\n"; } footer += "\t\n
\n"; // Write the footer out sw.Write(footer); sw.Flush(); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); return false; } return true; } } }