diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index d9132b61..b7a5bd93 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -5750,6 +5750,28 @@ namespace SabreTools.Library.DatFiles outfileNames.Add(DatFormat.OfflineList, CreateOutfileNamesHelper(outDir, ".ol.xml", overwrite)); } + // openMSX + if (((DatFormat & DatFormat.OpenMSX) != 0) + && (DatFormat & DatFormat.Logiqx) == 0 + && (DatFormat & DatFormat.LogiqxDepreciated) == 0 + && (DatFormat & DatFormat.Listxml) == 0 + && (DatFormat & DatFormat.SabreDat) == 0 + && (DatFormat & DatFormat.SoftwareList) == 0 + && (DatFormat & DatFormat.OfflineList) == 0) + { + outfileNames.Add(DatFormat.OfflineList, CreateOutfileNamesHelper(outDir, ".xml", overwrite)); + } + if (((DatFormat & DatFormat.OpenMSX) != 0 + && ((DatFormat & DatFormat.Logiqx) != 0 + || (DatFormat & DatFormat.LogiqxDepreciated) != 0 + || (DatFormat & DatFormat.Listxml) != 0 + || (DatFormat & DatFormat.SabreDat) != 0 + || (DatFormat & DatFormat.SoftwareList) != 0 + || (DatFormat & DatFormat.OfflineList) != 0))) + { + outfileNames.Add(DatFormat.OfflineList, CreateOutfileNamesHelper(outDir, ".msx.xml", overwrite)); + } + // Redump MD5 if ((DatFormat & DatFormat.RedumpMD5) != 0) { diff --git a/SabreTools.Library/DatFiles/OpenMSX.cs b/SabreTools.Library/DatFiles/OpenMSX.cs new file mode 100644 index 00000000..64db951e --- /dev/null +++ b/SabreTools.Library/DatFiles/OpenMSX.cs @@ -0,0 +1,781 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Web; +using System.Xml; + +using SabreTools.Library.Data; +using SabreTools.Library.DatItems; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using FileStream = System.IO.FileStream; +using StreamWriter = System.IO.StreamWriter; +#endif +using NaturalSort; + +namespace SabreTools.Library.DatFiles +{ + /// + /// Represents parsing and writing of a openMSX softawre list XML DAT + /// + /// TODO: Verify that all write for this DatFile type is correct + internal class OpenMSX : DatFile + { + /// + /// Constructor designed for casting a base DatFile + /// + /// Parent DatFile to copy from + public OpenMSX(DatFile datFile) + : base(datFile, cloneHeader: false) + { + } + + /// + /// Parse a openMSX softawre list XML DAT and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + /// + /// + public override void ParseFile( + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + // Prepare all internal variables + Encoding enc = Utilities.GetEncoding(filename); + XmlReader xtr = Utilities.GetXmlTextReader(filename); + + // If we got a null reader, just return + if (xtr == null) + { + return; + } + + // Otherwise, read the file to the end + try + { + xtr.MoveToContent(); + while (!xtr.EOF) + { + // We only want elements + if (xtr.NodeType != XmlNodeType.Element) + { + xtr.Read(); + continue; + } + + switch (xtr.Name) + { + case "softwaredb": + Name = (String.IsNullOrWhiteSpace(Name) ? "openMSX Software List" : Name); + Description = (String.IsNullOrWhiteSpace(Description) ? Name : Name); + // string timestamp = xtr.GetAttribute("timestamp"); // CDATA + xtr.Read(); + break; + // We want to process the entire subtree of the software + case "software": + ReadSoftware(xtr.ReadSubtree(), filename, sysid, srcid, keep, clean, remUnicode); + + // Skip the software now that we've processed it + xtr.Skip(); + break; + default: + xtr.Read(); + break; + } + } + } + catch (Exception ex) + { + Globals.Logger.Warning("Exception found while parsing '{0}': {1}", filename, ex); + + // For XML errors, just skip the affected node + xtr?.Read(); + } + + xtr.Dispose(); + } + + /// + /// Read software information + /// + /// XmlReader representing a machine block + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + private void ReadSoftware( + XmlReader reader, + + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + // If we have an empty machine, skip it + if (reader == null) + { + return; + } + + // Otherwise, add what is possible + reader.MoveToContent(); + + bool containsItems = false; + + // Create a new machine + Machine machine = new Machine(); + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the roms from the machine + switch (reader.Name) + { + case "title": + machine.Name = reader.ReadElementContentAsString(); + break; + case "genmsxid": + // string id = reader.ReadElementContentAsString(); + reader.Read(); + break; + case "system": + // string system = reader.ReadElementContentAsString(); + reader.Read(); + break; + case "company": + machine.Manufacturer = reader.ReadElementContentAsString(); + break; + case "year": + machine.Year = reader.ReadElementContentAsString(); + break; + case "country": + // string country = reader.ReadElementContentAsString(); + reader.Read(); + break; + case "dump": + containsItems = ReadDump(reader.ReadSubtree(), machine, filename, sysid, srcid, keep, clean, remUnicode); + + // Skip the dump now that we've processed it + reader.Skip(); + break; + default: + reader.Read(); + break; + } + } + + // If no items were found for this machine, add a Blank placeholder + if (!containsItems) + { + if (this.KeepEmptyGames) + { + Blank blank = new Blank() + { + SystemID = sysid, + System = filename, + SourceID = srcid, + }; + blank.CopyMachineInformation(machine); + + // Now process and add the rom + ParseAddHelper(blank, clean, remUnicode); + } + } + } + + /// + /// Read dump information + /// + /// XmlReader representing a part block + /// Machine information to pass to contained items + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + private bool ReadDump( + XmlReader reader, + Machine machine, + + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + string temptype = reader.Name; + bool containsItems = false; + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the elements from the dump + switch (reader.Name) + { + case "rom": + containsItems = ReadRom(reader.ReadSubtree(), machine, filename, sysid, srcid, keep, clean, remUnicode); + + // Skip the rom now that we've processed it + reader.Skip(); + break; + case "megarom": + containsItems = ReadMegaRom(reader.ReadSubtree(), machine, filename, sysid, srcid, keep, clean, remUnicode); + + // Skip the megarom now that we've processed it + reader.Skip(); + break; + case "sccpluscart": + containsItems = ReadSccPlusCart(reader.ReadSubtree(), machine, filename, sysid, srcid, keep, clean, remUnicode); + + // Skip the sccpluscart now that we've processed it + reader.Skip(); + break; + case "original": + // bool value = Utilities.GetYesNo(reader.GetAttribute("value"); + // string original = reader.ReadElementContentAsString(); + reader.Read(); + break; + default: + reader.Read(); + break; + } + } + + return containsItems; + } + + /// + /// Read rom information + /// + /// XmlReader representing a part block + /// Machine information to pass to contained items + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + private bool ReadRom( + XmlReader reader, + Machine machine, + + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + string hash = "", offset = "", type = "", remark = ""; + bool containsItems = false; + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the elements from the rom + switch (reader.Name) + { + case "hash": + containsItems = true; + hash = reader.ReadElementContentAsString(); + break; + case "start": + offset = reader.ReadElementContentAsString(); + break; + case "type": + type = reader.ReadElementContentAsString(); + break; + case "remark": + remark = reader.ReadElementContentAsString(); + break; + default: + reader.Read(); + break; + } + } + + // Create and add the new rom + Rom rom = new Rom + { + Name = machine.Name + (!String.IsNullOrWhiteSpace(remark) ? " " + remark : ""), + Offset = offset, + SHA1 = Utilities.CleanHashData(hash, Constants.SHA1Length), + }; + + rom.CopyMachineInformation(machine); + ParseAddHelper(rom, clean, remUnicode); + + return containsItems; + } + + /// + /// Read megarom information + /// + /// XmlReader representing a part block + /// Machine information to pass to contained items + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + private bool ReadMegaRom( + XmlReader reader, + Machine machine, + + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + string hash = "", offset = "", type = "", remark = ""; + bool containsItems = false; + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the elements from the dump + switch (reader.Name) + { + case "hash": + containsItems = true; + hash = reader.ReadElementContentAsString(); + break; + case "start": + offset = reader.ReadElementContentAsString(); + break; + case "type": + type = reader.ReadElementContentAsString(); + break; + case "remark": + remark = reader.ReadElementContentAsString(); + break; + default: + reader.Read(); + break; + } + } + + // Create and add the new rom + Rom rom = new Rom + { + Name = machine.Name + (!String.IsNullOrWhiteSpace(remark) ? " " + remark : ""), + Offset = offset, + SHA1 = Utilities.CleanHashData(hash, Constants.SHA1Length), + }; + + rom.CopyMachineInformation(machine); + ParseAddHelper(rom, clean, remUnicode); + + return containsItems; + } + + /// + /// Read sccpluscart information + /// + /// XmlReader representing a part block + /// Machine information to pass to contained items + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + /// True if game names are sanitized, false otherwise (default) + /// True if we should remove non-ASCII characters from output, false otherwise (default) + private bool ReadSccPlusCart( + XmlReader reader, + Machine machine, + + // Standard Dat parsing + string filename, + int sysid, + int srcid, + + // Miscellaneous + bool keep, + bool clean, + bool remUnicode) + { + string hash = "", boot = "", remark = ""; + bool containsItems = false; + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the elements from the dump + switch (reader.Name) + { + case "boot": + boot = reader.ReadElementContentAsString(); + break; + case "hash": + containsItems = true; + hash = reader.ReadElementContentAsString(); + break; + case "remark": + remark = reader.ReadElementContentAsString(); + break; + default: + reader.Read(); + break; + } + } + + // Create and add the new rom + Rom rom = new Rom + { + Name = machine.Name + (!String.IsNullOrWhiteSpace(remark) ? " " + remark : ""), + SHA1 = Utilities.CleanHashData(hash, Constants.SHA1Length), + }; + + rom.CopyMachineInformation(machine); + ParseAddHelper(rom, clean, remUnicode); + + return containsItems; + } + + /// + /// Create and open an output file for writing direct from a dictionary + /// + /// Name of the file to write to + /// True if blank roms should be skipped on output, false otherwise (default) + /// True if the DAT was written correctly, false otherwise + public override bool WriteToFile(string outfile, bool ignoreblanks = false) + { + try + { + Globals.Logger.User("Opening file for writing: {0}", outfile); + FileStream fs = Utilities.TryCreate(outfile); + + // If we get back null for some reason, just log and return + if (fs == null) + { + Globals.Logger.Warning("File '{0}' could not be created for writing! Please check to see if the file is writable", outfile); + return false; + } + + StreamWriter sw = new StreamWriter(fs, new UTF8Encoding(false)); + + // Write out the header + WriteHeader(sw); + + // Write out each of the machines and roms + string lastgame = null; + + // Get a properly sorted set of keys + List keys = Keys; + keys.Sort(new NaturalComparer()); + + foreach (string key in keys) + { + List roms = this[key]; + + // Resolve the names in the block + roms = DatItem.ResolveNames(roms); + + for (int index = 0; index < roms.Count; index++) + { + DatItem rom = roms[index]; + + // There are apparently times when a null rom can skip by, skip them + if (rom.Name == null || rom.MachineName == null) + { + Globals.Logger.Warning("Null rom found!"); + continue; + } + + // If we have a different game and we're not at the start of the list, output the end of last item + if (lastgame != null && lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant()) + { + WriteEndGame(sw); + } + + // If we have a new game, output the beginning of the new item + if (lastgame == null || lastgame.ToLowerInvariant() != rom.MachineName.ToLowerInvariant()) + { + WriteStartGame(sw, rom); + } + + // If we have a "null" game (created by DATFromDir or something similar), log it to file + if (rom.Type == ItemType.Rom + && ((Rom)rom).Size == -1 + && ((Rom)rom).CRC == "null") + { + Globals.Logger.Verbose("Empty folder found: {0}", rom.MachineName); + + lastgame = rom.MachineName; + continue; + } + + // Now, output the rom data + WriteDatItem(sw, rom, ignoreblanks); + + // Set the new data to compare against + lastgame = rom.MachineName; + } + } + + // Write the file footer out + WriteFooter(sw); + + Globals.Logger.Verbose("File written!" + Environment.NewLine); + sw.Dispose(); + fs.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Write out DAT header using the supplied StreamWriter + /// + /// StreamWriter to output to + /// True if the data was written, false on error + private bool WriteHeader(StreamWriter sw) + { + try + { + string header = "\n" + + "\n" + + "\n" + + @" +"; + + // Write the header out + sw.Write(header); + sw.Flush(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Write out Game start using the supplied StreamWriter + /// + /// StreamWriter to output to + /// DatItem object to be output + /// True if the data was written, false on error + private bool WriteStartGame(StreamWriter sw, DatItem rom) + { + try + { + // No game should start with a path separator + if (rom.MachineName.StartsWith(Path.DirectorySeparatorChar.ToString())) + { + rom.MachineName = rom.MachineName.Substring(1); + } + + string state = "\n" + + "\t" + HttpUtility.HtmlEncode(rom.MachineName) + "\n" + // + "\t" + msxid + "\n" + // + "\t" + system + "\n" + + "\t" + rom.Manufacturer + "\n" + + "\t" + rom.Year + "\n"; + // + "\t" + rom.Year + "\n"; + + sw.Write(state); + sw.Flush(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Write out Game start using the supplied StreamWriter + /// + /// StreamWriter to output to + /// True if the data was written, false on error + private bool WriteEndGame(StreamWriter sw) + { + try + { + string state = "\n"; + + sw.Write(state); + sw.Flush(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Write out DatItem using the supplied StreamWriter + /// + /// StreamWriter to output to + /// DatItem object to be output + /// True if blank roms should be skipped on output, false otherwise (default) + /// True if the data was written, false on error + private bool WriteDatItem(StreamWriter sw, DatItem rom, bool ignoreblanks = false) + { + // If we are in ignore blanks mode AND we have a blank (0-size) rom, skip + if (ignoreblanks + && (rom.Type == ItemType.Rom + && (((Rom)rom).Size == 0 || ((Rom)rom).Size == -1))) + { + return true; + } + + try + { + string state = ""; + + // Pre-process the item name + ProcessItemName(rom, true); + + switch (rom.Type) + { + case ItemType.Archive: + break; + case ItemType.BiosSet: + break; + case ItemType.Disk: + break; + case ItemType.Release: + break; + case ItemType.Rom: // Currently this encapsulates rom, megarom, and sccpluscart + state += "\t\t" + // + "GoodMSX" + + "" + + (!String.IsNullOrWhiteSpace(((Rom)rom).Offset) ? "" + ((Rom)rom).Offset + "" : "") + // + "Normal" + + "" + ((Rom)rom).SHA1 + "" + // + "" + + "\n"; + break; + case ItemType.Sample: + break; + } + + sw.Write(state); + sw.Flush(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Write out DAT footer using the supplied StreamWriter + /// + /// StreamWriter to output to + /// True if the data was written, false on error + private bool WriteFooter(StreamWriter sw) + { + try + { + string footer = "\n\n"; + + // Write the footer out + sw.Write(footer); + sw.Flush(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + } +} diff --git a/SabreTools.Library/Data/Constants.cs b/SabreTools.Library/Data/Constants.cs index 85df7db6..fa5cd0be 100644 --- a/SabreTools.Library/Data/Constants.cs +++ b/SabreTools.Library/Data/Constants.cs @@ -424,6 +424,16 @@ namespace SabreTools.Library.Data +"; + public const string OpenMSXDTD = @" + + + + + + + + "; public const string SoftwareListDTD = @" diff --git a/SabreTools.Library/Data/Flags.cs b/SabreTools.Library/Data/Flags.cs index 87bbf919..f990e409 100644 --- a/SabreTools.Library/Data/Flags.cs +++ b/SabreTools.Library/Data/Flags.cs @@ -199,9 +199,10 @@ namespace SabreTools.Library.Data Listxml = SoftwareList << 1, OfflineList = Listxml << 1, SabreDat = OfflineList << 1, + OpenMSX = SabreDat << 1, // Propietary Formats - ClrMamePro = SabreDat << 1, + ClrMamePro = OpenMSX << 1, RomCenter = ClrMamePro << 1, DOSCenter = RomCenter << 1, AttractMode = DOSCenter << 1, diff --git a/SabreTools.Library/README.1ST b/SabreTools.Library/README.1ST index b9f398b9..b5e5bf40 100644 --- a/SabreTools.Library/README.1ST +++ b/SabreTools.Library/README.1ST @@ -218,6 +218,7 @@ Options: lx, listxml - MAME Listxml miss, missfile - GoodTools Missfile md5 - MD5 + msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter sd, sabredat - SabreDat XML @@ -402,6 +403,7 @@ Options: lx, listxml - MAME Listxml miss, missfile - GoodTools Missfile md5 - MD5 + msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter sd, sabredat - SabreDat XML @@ -700,27 +702,28 @@ Options: this flag are allowed. Possible values are: - all - All available DAT types - am, attractmode - AttractMode XML - cmp, clrmamepro - ClrMamePro - csv - Standardized Comma-Separated Value - dc, doscenter - DOSCenter - lr, listrom - MAME Listrom - lx, listxml - MAME Listxml - miss, missfile - GoodTools Missfile - md5 - MD5 - ol, offlinelist - OfflineList XML - rc, romcenter - RomCenter - sd, sabredat - SabreDat XML - sfv - SFV - sha1 - SHA1 - sha256 - SHA256 - sha384 - SHA384 - sha512 - SHA512 - sl, softwarelist - MAME Software List XML - ssv - Standardized Semicolon-Separated Value - tsv - Standardized Tab-Separated Value - xml, logiqx - Logiqx XML + all - All available DAT types + am, attractmode - AttractMode XML + cmp, clrmamepro - ClrMamePro + csv - Standardized Comma-Separated Value + dc, doscenter - DOSCenter + lr, listrom - MAME Listrom + lx, listxml - MAME Listxml + miss, missfile - GoodTools Missfile + md5 - MD5 + msx, openmsx - openMSX Software List + ol, offlinelist - OfflineList XML + rc, romcenter - RomCenter + sd, sabredat - SabreDat XML + sfv - SFV + sha1 - SHA1 + sha256 - SHA256 + sha384 - SHA384 + sha512 - SHA512 + sl, softwarelist - MAME Software List XML + ssv - Standardized Semicolon-Separated Value + tsv - Standardized Tab-Separated Value + xml, logiqx - Logiqx XML -pre, --prefix Set prefix for all lines Set a generic prefix to be prepended to all outputted lines. diff --git a/SabreTools.Library/SabreTools.Library.csproj b/SabreTools.Library/SabreTools.Library.csproj index 75ddf549..94d56ad3 100644 --- a/SabreTools.Library/SabreTools.Library.csproj +++ b/SabreTools.Library/SabreTools.Library.csproj @@ -99,6 +99,7 @@ + diff --git a/SabreTools.Library/Tools/Utilities.cs b/SabreTools.Library/Tools/Utilities.cs index 356f2c51..46872a4a 100644 --- a/SabreTools.Library/Tools/Utilities.cs +++ b/SabreTools.Library/Tools/Utilities.cs @@ -633,6 +633,8 @@ namespace SabreTools.Library.Tools return new Missfile(baseDat); case DatFormat.OfflineList: return new OfflineList(baseDat); + case DatFormat.OpenMSX: + return new OpenMSX(baseDat); case DatFormat.RedumpMD5: return new Hashfile(baseDat, Hash.MD5); case DatFormat.RedumpSFV: @@ -742,11 +744,14 @@ namespace SabreTools.Library.Tools case "lx": case "listxml": return DatFormat.Listxml; + case "md5": + return DatFormat.RedumpMD5; case "miss": case "missfile": return DatFormat.MissFile; - case "md5": - return DatFormat.RedumpMD5; + case "msx": + case "openmsx": + return DatFormat.OpenMSX; case "ol": case "offlinelist": return DatFormat.OfflineList; @@ -1182,6 +1187,10 @@ namespace SabreTools.Library.Tools { return DatFormat.Listxml; } + else if (second.StartsWith("