[ALL] Fix device merging, add new parsing prototypes

SabreTools.Library/DatFiles/SoftwareList.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 using System.Web;
+using System.Xml;
 
 using SabreTools.Library.Data;
 using SabreTools.Library.DatItems;
@@ -20,8 +21,9 @@ using NaturalSort;
 namespace SabreTools.Library.DatFiles
 {
 	/// <summary>
-	/// Represents parsing and writing of an SabreDat XML DAT
+	/// Represents parsing and writing of a SofwareList, M1, or MAME XML DAT
 	/// </summary>
+	/// TODO: Verify that all read/write for this DatFile type is correct
 	internal class SoftwareList : DatFile
 	{
 		/// <summary>
@@ -34,7 +36,7 @@ namespace SabreTools.Library.DatFiles
 		}
 
 		/// <summary>
-		/// Parse an SabreDat XML DAT and return all found games and roms within
+		/// Parse an SofwareList XML DAT and return all found games and roms within
 		/// </summary>
 		/// <param name="filename">Name of the file to be parsed</param>
 		/// <param name="sysid">System ID for the DAT</param>
@@ -56,7 +58,381 @@ namespace SabreTools.Library.DatFiles
 			bool remUnicode)
 		{
 			// All XML-derived DATs share a lot in common so it just calls one implementation
-			new Logiqx(this, false).ParseFile(filename, sysid, srcid, keep, clean, remUnicode);
+			// new Logiqx(this, false).ParseFile(filename, sysid, srcid, keep, clean, remUnicode);
+
+			// Prepare all internal variables
+			Encoding enc = Utilities.GetEncoding(filename);
+			XmlReader xtr = Utilities.GetXmlTextReader(filename);
+
+			// If we got a null reader, just return
+			if (xtr == null)
+			{
+				return;
+			}
+
+			// Otherwise, read the file to the end
+			try
+			{
+				xtr.MoveToContent();
+				while (!xtr.EOF)
+				{
+					// We only want elements
+					if (xtr.NodeType != XmlNodeType.Element)
+					{
+						xtr.Read();
+						continue;
+					}
+
+					switch (xtr.Name)
+					{
+						case "softwarelist":
+							Name = (String.IsNullOrWhiteSpace(Name) ? xtr.GetAttribute("name") ?? "" : Name);
+							Description = (String.IsNullOrWhiteSpace(Description) ? xtr.GetAttribute("description") ?? "" : Description);
+							if (ForceMerging == ForceMerging.None)
+							{
+								ForceMerging = Utilities.GetForceMerging(xtr.GetAttribute("forcemerging"));
+							}
+							if (ForceNodump == ForceNodump.None)
+							{
+								ForceNodump = Utilities.GetForceNodump(xtr.GetAttribute("forcenodump"));
+							}
+							if (ForcePacking == ForcePacking.None)
+							{
+								ForcePacking = Utilities.GetForcePacking(xtr.GetAttribute("forcepacking"));
+							}
+							xtr.Read();
+							break;
+						// We want to process the entire subtree of the machine
+						case "software":
+							ReadSoftware(xtr.ReadSubtree(), filename, sysid, srcid, keep, clean, remUnicode);
+
+							// Skip the software now that we've processed it
+							xtr.Skip();
+							break;
+						default:
+							xtr.Read();
+							break;
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				Globals.Logger.Warning("Exception found while parsing '{0}': {1}", filename, ex);
+
+				// For XML errors, just skip the affected node
+				xtr?.Read();
+			}
+
+			xtr.Dispose();
+		}
+
+		/// <summary>
+		/// Read software information
+		/// </summary>
+		/// <param name="filename">Name of the file to be parsed</param>
+		/// <param name="sysid">System ID for the DAT</param>
+		/// <param name="srcid">Source ID for the DAT</param>
+		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
+		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		private void ReadSoftware(
+			XmlReader reader,
+
+			// Standard Dat parsing
+			string filename,
+			int sysid,
+			int srcid,
+
+			// Miscellaneous
+			bool keep,
+			bool clean,
+			bool remUnicode)
+		{
+			// If we have an empty software, skip it
+			if (reader == null)
+			{
+				return;
+			}
+
+			// Otherwise, add what is possible
+			reader.MoveToContent();
+
+			string key = "";
+			string temptype = reader.Name;
+			bool containsItems = false;
+
+			// Create a new machine
+			MachineType machineType = MachineType.NULL;
+			if (Utilities.GetYesNo(reader.GetAttribute("isbios")) == true)
+			{
+				machineType |= MachineType.Bios;
+			}
+			if (Utilities.GetYesNo(reader.GetAttribute("isdevice")) == true)
+			{
+				machineType |= MachineType.Device;
+			}
+			if (Utilities.GetYesNo(reader.GetAttribute("ismechanical")) == true)
+			{
+				machineType |= MachineType.Mechanical;
+			}
+
+			Machine machine = new Machine
+			{
+				Name = reader.GetAttribute("name"),
+				Description = reader.GetAttribute("name"),
+				Supported = Utilities.GetYesNo(reader.GetAttribute("supported")), // (yes|partial|no) "yes"
+
+				CloneOf = reader.GetAttribute("cloneof") ?? "",
+
+				MachineType = (machineType == MachineType.NULL ? MachineType.None : machineType),
+			};
+
+			while (!reader.EOF)
+			{
+				// We only want elements
+				if (reader.NodeType != XmlNodeType.Element)
+				{
+					reader.Read();
+					continue;
+				}
+
+				// Get the elements from the software
+				switch (reader.Name)
+				{
+					case "description":
+						machine.Description = reader.ReadElementContentAsString();
+						break;
+					case "year":
+						machine.Year = reader.ReadElementContentAsString();
+						break;
+					case "publisher":
+						machine.Publisher = reader.ReadElementContentAsString();
+						break;
+					case "info":
+						machine.Infos.Add(new Tuple<string, string>(reader.GetAttribute("name"), reader.GetAttribute("value")));
+
+						reader.Read();
+						break;
+					case "sharedfeat":
+						// string name = reader.GetAttribute("name");
+						// string value = reader.GetAttribute("value");
+
+						reader.Read();
+						break;
+					case "part": // Contains all rom and disk information
+						containsItems = ReadPart(reader.ReadSubtree(), machine, filename, sysid, srcid, keep, clean, remUnicode);
+
+						// Skip the part now that we've processed it
+						reader.Skip();
+						break;
+					default:
+						reader.Read();
+						break;
+				}
+			}
+
+			// If no items were found for this machine, add a Blank placeholder
+			if (!containsItems)
+			{
+				Blank blank = new Blank()
+				{
+					SystemID = sysid,
+					System = filename,
+					SourceID = srcid,
+				};
+
+				blank.CopyMachineInformation(machine);
+
+				// Now process and add the rom
+				key = ParseAddHelper(blank, clean, remUnicode);
+			}
+		}
+
+		/// <summary>
+		/// Read part information
+		/// </summary>
+		/// <param name="filename">Name of the file to be parsed</param>
+		/// <param name="sysid">System ID for the DAT</param>
+		/// <param name="srcid">Source ID for the DAT</param>
+		/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
+		/// <param name="clean">True if game names are sanitized, false otherwise (default)</param>
+		/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
+		private bool ReadPart(
+			XmlReader reader,
+			Machine machine,
+
+			// Standard Dat parsing
+			string filename,
+			int sysid,
+			int srcid,
+
+			// Miscellaneous
+			bool keep,
+			bool clean,
+			bool remUnicode)
+		{
+			string key = "", areaname = "", partname = "", partinterface = "";
+			string temptype = reader.Name;
+			long? areasize = null;
+			List<Tuple<string, string>> features = new List<Tuple<string, string>>();
+			bool containsItems = false;
+
+			while (!reader.EOF)
+			{
+				// We only want elements
+				if (reader.NodeType != XmlNodeType.Element)
+				{
+					if (reader.NodeType == XmlNodeType.EndElement && reader.Name == "part")
+					{
+						partname = "";
+						partinterface = "";
+						features = new List<Tuple<string, string>>();
+					}
+					if (reader.NodeType == XmlNodeType.EndElement && (reader.Name == "dataarea" || reader.Name == "diskarea"))
+					{
+						areaname = "";
+						areasize = null;
+					}
+
+					reader.Read();
+					continue;
+				}
+
+				// Get the elements from the software
+				switch (reader.Name)
+				{
+					case "part":
+						partname = reader.GetAttribute("name");
+						partinterface = reader.GetAttribute("interface");
+
+						reader.Read();
+						break;
+					case "feature":
+						features.Add(new Tuple<string, string>(reader.GetAttribute("name"), reader.GetAttribute("feature")));
+
+						reader.Read();
+						break;
+					case "dataarea":
+						areaname = reader.GetAttribute("name");
+						if (reader.GetAttribute("size") != null)
+						{
+							if (Int64.TryParse(reader.GetAttribute("size"), out long tempas))
+							{
+								areasize = tempas;
+							}
+						}
+						// string width = reader.GetAttribute("width"); // (8|16|32|64) "8"
+						// string endianness = reader.GetAttribute("endianness"); // endianness (big|little) "little"
+						
+						reader.Read();
+						break;
+					case "rom": // TODO: Put this in a proper subreader under dataarea
+						containsItems = true;
+
+						// If the rom is continue or ignore, add the size to the previous rom
+						if (reader.GetAttribute("loadflag") == "continue" || reader.GetAttribute("loadflag") == "ignore")
+						{
+							int index = this[key].Count - 1;
+							DatItem lastrom = this[key][index];
+							if (lastrom.Type == ItemType.Rom)
+							{
+								((Rom)lastrom).Size += Utilities.GetSize(reader.GetAttribute("size"));
+							}
+							this[key].RemoveAt(index);
+							this[key].Add(lastrom);
+							reader.Read();
+							continue;
+						}
+
+						DatItem rom = new Rom
+						{
+							Name = reader.GetAttribute("name"),
+							Size = Utilities.GetSize(reader.GetAttribute("size")),
+							CRC = reader.GetAttribute("crc")?.ToLowerInvariant(),
+							MD5 = reader.GetAttribute("md5")?.ToLowerInvariant(),
+							SHA1 = reader.GetAttribute("sha1")?.ToLowerInvariant(),
+							SHA256 = reader.GetAttribute("sha256")?.ToLowerInvariant(),
+							SHA384 = reader.GetAttribute("sha384")?.ToLowerInvariant(),
+							SHA512 = reader.GetAttribute("sha512")?.ToLowerInvariant(),
+							Offset = reader.GetAttribute("offset"),
+							// Value = reader.GetAttribute("value");
+							ItemStatus = Utilities.GetItemStatus(reader.GetAttribute("status")),
+							// LoadFlag = reader.GetAttribute("loadflag"), // (load16_byte|load16_word|load16_word_swap|load32_byte|load32_word|load32_word_swap|load32_dword|load64_word|load64_word_swap|reload|fill|continue|reload_plain|ignore)
+
+							AreaName = areaname,
+							AreaSize = areasize,
+							Features = features,
+							PartName = partname,
+							PartInterface = partinterface,
+
+							SystemID = sysid,
+							System = filename,
+							SourceID = srcid,
+						};
+
+						rom.CopyMachineInformation(machine);
+
+						// Now process and add the rom
+						key = ParseAddHelper(rom, clean, remUnicode);
+
+						reader.Read();
+						break;
+					case "diskarea":
+						areaname = reader.GetAttribute("name");
+
+						reader.Read();
+						break;
+					case "disk": // TODO: Put this in a proper subreader under diskarea
+						containsItems = true;
+
+						DatItem disk = new Disk
+						{
+							Name = reader.GetAttribute("name"),
+							MD5 = reader.GetAttribute("md5")?.ToLowerInvariant(),
+							SHA1 = reader.GetAttribute("sha1")?.ToLowerInvariant(),
+							SHA256 = reader.GetAttribute("sha256")?.ToLowerInvariant(),
+							SHA384 = reader.GetAttribute("sha384")?.ToLowerInvariant(),
+							SHA512 = reader.GetAttribute("sha512")?.ToLowerInvariant(),
+							ItemStatus = Utilities.GetItemStatus(reader.GetAttribute("status")),
+							Writable = Utilities.GetYesNo(reader.GetAttribute("writable")),
+
+							AreaName = areaname,
+							AreaSize = areasize,
+							Features = features,
+							PartName = partname,
+							PartInterface = partinterface,
+
+							SystemID = sysid,
+							System = filename,
+							SourceID = srcid,
+						};
+
+						disk.CopyMachineInformation(machine);
+
+						// Now process and add the rom
+						key = ParseAddHelper(disk, clean, remUnicode);
+
+						reader.Read();
+						break;
+					case "dipswitch":
+						// string name = reader.GetAttribute("name");
+						// string tag = reader.GetAttribute("tag");
+						// string mask = reader.GetAttribute("mask");
+
+						// For every <dipvalue> element...
+						// string name = reader.GetAttribute("name");
+						// string value = reader.GetAttribute("value");
+						// bool? default = Utilities.GetYesNo(reader.GetAttribute("default")); // (yes|no) "no"
+
+						reader.Read();
+						break;
+					default:
+						reader.Read();
+						break;
+				}
+			}
+
+			return containsItems;
 		}
 
 		/// <summary>
@@ -210,18 +586,12 @@ namespace SabreTools.Library.DatFiles
 				}
 
 				string state = "\t<software name=\"" + HttpUtility.HtmlEncode(rom.MachineName) + "\""
-							+ (rom.Supported != null ? " supported=\"" + (rom.Supported == true ? "yes" : "no") + "\"" : "") +
-							(ExcludeOf ? "" :
+							+ (ExcludeOf ? "" :
 									(String.IsNullOrWhiteSpace(rom.CloneOf) || (rom.MachineName.ToLowerInvariant() == rom.CloneOf.ToLowerInvariant())
 										? ""
-										: " cloneof=\"" + HttpUtility.HtmlEncode(rom.CloneOf) + "\"") +
-									(String.IsNullOrWhiteSpace(rom.RomOf) || (rom.MachineName.ToLowerInvariant() == rom.RomOf.ToLowerInvariant())
-										? ""
-										: " romof=\"" + HttpUtility.HtmlEncode(rom.RomOf) + "\"") +
-									(String.IsNullOrWhiteSpace(rom.SampleOf) || (rom.MachineName.ToLowerInvariant() == rom.SampleOf.ToLowerInvariant())
-										? ""
-										: " sampleof=\"" + HttpUtility.HtmlEncode(rom.SampleOf) + "\"")
-								) + ">\n"
+										: " cloneof=\"" + HttpUtility.HtmlEncode(rom.CloneOf) + "\"")
+								)
+							+ " supported=\"" + (rom.Supported == true ? "yes" : rom.Supported == false ? "no" : "partial") + "\">\n"
 							+ "\t\t<description>" + HttpUtility.HtmlEncode(rom.MachineDescription) + "</description>\n"
 							+ (rom.Year != null ? "\t\t<year>" + HttpUtility.HtmlEncode(rom.Year) + "</year>\n" : "")
 							+ (rom.Publisher != null ? "\t\t<publisher>" + HttpUtility.HtmlEncode(rom.Publisher) + "</publisher>\n" : "");
@@ -295,24 +665,6 @@ namespace SabreTools.Library.DatFiles
 
 				switch (rom.Type)
 				{
-					case ItemType.Archive:
-						state += "\t\t\t<dataarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "archive" : rom.AreaName) + "\""
-								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
-							+ "\t\t\t\t<archive name=\"" + HttpUtility.HtmlEncode(rom.Name) + "\""
-							+ "/>\n"
-							+ "\t\t\t</dataarea>\n";
-						break;
-					case ItemType.BiosSet:
-						state += "\t\t\t<dataarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "biosset" : rom.AreaName) + "\""
-								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
-							+ "\t\t\t\t<biosset name\"" + HttpUtility.HtmlEncode(rom.Name) + "\""
-							+ (!String.IsNullOrWhiteSpace(((BiosSet)rom).Description) ? " description=\"" + HttpUtility.HtmlEncode(((BiosSet)rom).Description) + "\"" : "")
-							+ (((BiosSet)rom).Default != null
-								? ((BiosSet)rom).Default.ToString().ToLowerInvariant()
-								: "")
-							+ "/>\n"
-							+ "\t\t\t</dataarea>\n";
-						break;
 					case ItemType.Disk:
 						state += "\t\t\t<diskarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "cdrom" : rom.AreaName) + "\""
 								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
@@ -323,22 +675,10 @@ namespace SabreTools.Library.DatFiles
 							+ (!String.IsNullOrWhiteSpace(((Disk)rom).SHA384) ? " sha384=\"" + ((Disk)rom).SHA384.ToLowerInvariant() + "\"" : "")
 							+ (!String.IsNullOrWhiteSpace(((Disk)rom).SHA512) ? " sha512=\"" + ((Disk)rom).SHA512.ToLowerInvariant() + "\"" : "")
 							+ (((Disk)rom).ItemStatus != ItemStatus.None ? " status=\"" + ((Disk)rom).ItemStatus.ToString().ToLowerInvariant() + "\"" : "")
+							+ (((Disk)rom).Writable != null ? " writable=\"" + (((Disk)rom).Writable == true ? "yes" : "no") + "\"" : "")
 							+ "/>\n"
 							+ "\t\t\t</diskarea>\n";
 						break;
-					case ItemType.Release:
-						state += "\t\t\t<dataarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "release" : rom.AreaName) + "\""
-								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
-							+ "\t\t\t\t<release name\"" + HttpUtility.HtmlEncode(rom.Name) + "\""
-							+ (!String.IsNullOrWhiteSpace(((Release)rom).Region) ? " region=\"" + HttpUtility.HtmlEncode(((Release)rom).Region) + "\"" : "")
-							+ (!String.IsNullOrWhiteSpace(((Release)rom).Language) ? " language=\"" + HttpUtility.HtmlEncode(((Release)rom).Language) + "\"" : "")
-							+ (!String.IsNullOrWhiteSpace(((Release)rom).Date) ? " date=\"" + HttpUtility.HtmlEncode(((Release)rom).Date) + "\"" : "")
-							+ (((Release)rom).Default != null
-								? ((Release)rom).Default.ToString().ToLowerInvariant()
-								: "")
-							+ "/>\n"
-							+ "\t\t\t</dataarea>\n";
-						break;
 					case ItemType.Rom:
 						state += "\t\t\t<dataarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "rom" : rom.AreaName) + "\""
 								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
@@ -350,15 +690,10 @@ namespace SabreTools.Library.DatFiles
 							+ (!String.IsNullOrWhiteSpace(((Rom)rom).SHA256) ? " sha256=\"" + ((Rom)rom).SHA256.ToLowerInvariant() + "\"" : "")
 							+ (!String.IsNullOrWhiteSpace(((Rom)rom).SHA384) ? " sha384=\"" + ((Rom)rom).SHA384.ToLowerInvariant() + "\"" : "")
 							+ (!String.IsNullOrWhiteSpace(((Rom)rom).SHA512) ? " sha512=\"" + ((Rom)rom).SHA512.ToLowerInvariant() + "\"" : "")
-							+ (!String.IsNullOrWhiteSpace(((Rom)rom).Date) ? " date=\"" + ((Rom)rom).Date + "\"" : "")
+							+ (!String.IsNullOrWhiteSpace(((Rom)rom).Offset) ? " offset=\"" + ((Rom)rom).Offset + "\"" : "")
+							// + (!String.IsNullOrWhiteSpace(((Rom)rom).Value) ? " value=\"" + ((Rom)rom).Value + "\"" : "")
 							+ (((Rom)rom).ItemStatus != ItemStatus.None ? " status=\"" + ((Rom)rom).ItemStatus.ToString().ToLowerInvariant() + "\"" : "")
-							+ "/>\n"
-							+ "\t\t\t</dataarea>\n";
-						break;
-					case ItemType.Sample:
-						state += "\t\t\t<dataarea name=\"" + (String.IsNullOrWhiteSpace(rom.AreaName) ? "sample" : rom.AreaName) + "\""
-								+ (rom.AreaSize != null ? " size=\"" + rom.AreaSize + "\"" : "") + ">\n"
-							+ "\t\t\t\t<sample type=\"sample\" name=\"" + HttpUtility.HtmlEncode(rom.Name) + "\""
+							// + (!String.IsNullOrWhiteSpace(((Rom)rom).Loadflag) ? " loadflag=\"" + ((Rom)rom).Loadflag + "\"" : "")
 							+ "/>\n"
 							+ "\t\t\t</dataarea>\n";
 						break;