Simplified importing DATs into the database by first converting all of them to XML and then having a unified parser. As a consequence of this, there is also the ability to directly convert one RV DAT to an equivalent XML DAT now.

DATabase/Converters.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Xml.Linq;
+
+namespace DATabase
+{
+	class Converters
+	{
+		private static string _headerPattern = @"(^.*?) \($";
+		private static string _romPattern = @"^\s+((?:rom)|(?:disk)) \( (name) ""(.*?)"" (?:(size) (.*?) )?(?:(crc) (.*?))?(?:(md5) (.*?) )?(?:(sha1) (.*?) )?\)";
+		private static string _itemPattern = @"^\s+(.*?) ""(.*?)""";
+		private static string _endPattern = @"^\s*\)\s*$";
+
+		public static XElement RomVaultToXML (string[] filecontents)
+		{
+			XElement elem = new XElement("datafile");
+
+			bool block = false;
+			for (int k = 0; k < filecontents.Length; k++)
+			{
+				string line = filecontents[k];
+
+				// If the line is the header or a game
+				if (Regex.IsMatch(line, _headerPattern))
+				{
+					GroupCollection gc = Regex.Match(line, _headerPattern).Groups;
+
+					if (gc[1].Value == "clrmamepro" || gc[1].Value == "romvault")
+					{
+						elem.Add(new XElement("header"));
+						elem = elem.Elements("header").Last();
+					}
+					else
+					{
+						elem.Add(new XElement(gc[1].Value));
+						elem = elem.Elements(gc[1].Value).Last();
+					}
+					
+					block = true;
+				}
+
+				// If the line is a rom or disk and we're in a block
+				else if (Regex.IsMatch(line, _romPattern) && block)
+			    {
+					GroupCollection gc = Regex.Match(line, _romPattern).Groups;
+
+					XElement temp = new XElement(gc[1].Value);
+
+					// Loop over all attributes and add them if possible
+					for (int i = 1; i < gc.Count; i++)
+					{
+						if (i + 2 < gc.Count)
+						{
+							temp.SetAttributeValue(gc[i+1].Value, gc[i+2].Value);
+							i++;
+						}
+					}
+
+					elem.Add(new XElement(temp));
+				}
+				// If the line is anything but a rom or disk  and we're in a block
+				else if (Regex.IsMatch(line, _itemPattern) && block)
+				{
+					GroupCollection gc = Regex.Match(line, _itemPattern).Groups;
+
+					if (gc[1].Value == "name" && elem.Name != "header")
+					{
+						elem.SetAttributeValue(gc[1].Value, gc[2].Value);
+						elem.Add(new XElement("description", gc[2].Value));
+					}
+					else
+					{
+						elem.Add(new XElement(gc[1].Value, gc[2].Value));
+					}
+				}
+
+				// If we find an end bracket that's not associated with anything else, the block is done
+				else if (Regex.IsMatch(line, _endPattern) && block)
+			    {
+					block = false;
+					elem = elem.Parent;
+				}
+			}
+
+			return elem;
+		}
+	}
+}

DATabase/DATabase.csproj

@@ -66,6 +66,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Converters.cs" />
     <Compile Include="Generate.cs" />
     <Compile Include="Importer.cs" />
     <Compile Include="Program.cs" />

DATabase/Importer.cs

@@ -258,142 +258,81 @@ namespace DATabase
 				FileStream fs = File.OpenRead(_filepath);
 				StreamReader sr = new StreamReader(fs);
 
-				Console.WriteLine("got here");
+				XmlDocument doc = new XmlDocument();
-
+				try
-				// Set necessary dat values
-				string format = "";
-				bool machinefound = false;
-				string machinename = "";
-				string description = "";
-				long gameid = 0;
-				bool comment = false;
-
-				// Parse the file for its rom information
-				while (sr.Peek() > 0)
 				{
-					string line = sr.ReadLine();
+					doc.LoadXml(sr.ReadToEnd());
+				}
+				catch (XmlException ex)
+				{
+					doc.LoadXml(Converters.RomVaultToXML(File.ReadAllLines(_filepath)).ToString());
+				}
 
-					// First each string has to be normalized
+				// Experimental looping using only XML parsing
-					line = Style.NormalizeChars(line);
+				XmlNode node = doc.FirstChild.FirstChild;
+				if (node.Name == "header")
+				{
+					node = node.NextSibling;
+                }
 
-					// If the input style hasn't been set, set it according to the header
+				while (node != null)
-					if (format == "")
+				{
+					if (node.Name == "machine" || node.Name == "game" || node.Name == "software")
 					{
-						if (line.IndexOf("<?xml version=\"1.0\" encoding=\"utf-8\"?>") != -1)
+						long gameid = AddGame(sysid, node.Attributes["name"].Value, srcid);
-                        {
+
-							format = "logiqx";
+						// Get the roms from the machine
-						}
+						if (node.HasChildNodes)
-						else if (line.IndexOf("clrmamepro (") != -1 || line.IndexOf("romvault (") != -1)
 						{
-							format = "romvault";
+							// If this node has children, traverse the children
-						}
+							foreach (XmlNode child in node.ChildNodes)
-					}
+							{
-					else if (line.IndexOf("<!DOCTYPE softwarelist") != -1)
+								// If we find a rom or disk, add it
-					{
+								if (child.Name == "rom" || child.Name == "disk")
-						format = "softwarelist";
+								{
-					}
+									AddRomHelper(
-
+										child.Name,
-					// If there's an XML-style comment, stop the presses and skip until it's over
+										gameid,
-					else if (line.IndexOf("-->") != -1)
+										child.Attributes["name"].Value,
-					{
+										date,
-						comment = false;
+										(child.Attributes["size"].Value != "" ? Int32.Parse(child.Attributes["size"].Value) : -1),
-					}
+										(child.Attributes["crc"].Value != "" ? child.Attributes["crc"].Value : ""),
-					else if (line.IndexOf("<!--") != -1)
+										(child.Attributes["md5"].Value != "" ? child.Attributes["md5"].Value : ""),
-					{
+										(child.Attributes["sha1"].Value != "" ? child.Attributes["sha1"].Value : "")
-						comment = true;
+                                    );
-					}
+								}
-
+								// If we find the signs of a software list, traverse the children
-					// Process Logiqx XML-derived DATs
+								else if (child.Name == "part" && child.HasChildNodes)
-					else if(format == "logiqx" && !comment)
+								{
-					{
+									foreach (XmlNode part in child.ChildNodes)
-						if (line.IndexOf("<machine") != -1 || line.IndexOf("<game") != -1)
+									{
-						{
+										// If we find a dataarea, traverse the children
-							machinefound = true;
+										if (part.Name == "dataarea")
-
+										{
-							XElement xml = XElement.Parse(line + (line.IndexOf("<machine") != -1 ? "</machine>" : "</game>"));
+											foreach (XmlNode data in part.ChildNodes)
-							machinename = xml.Attribute("name").Value;
+											{
-
+												// If we find a rom or disk, add it
-							gameid = AddGame(sysid, machinename, srcid);
+												if (data.Name == "rom" || data.Name == "disk")
-						}
+												{
-						else if (line.IndexOf("<rom") != -1 && machinefound)
+													AddRomHelper(
-						{
+														data.Name,
-							AddRom(line, machinename, "rom", gameid, date);
+														gameid,
-						}
+														data.Attributes["name"].Value,
-						else if (line.IndexOf("<disk") != -1 && machinefound)
+														date,
-						{
+														(data.Attributes["size"].Value != "" ? Int32.Parse(data.Attributes["size"].Value) : -1),
-							AddRom(line, machinename, "disk", gameid, date);
+														(data.Attributes["crc"].Value != "" ? data.Attributes["crc"].Value : ""),
-						}
+														(data.Attributes["md5"].Value != "" ? data.Attributes["md5"].Value : ""),
-						else if (line.IndexOf("</machine>") != -1 || line.IndexOf("</game>") != -1)
+														(data.Attributes["sha1"].Value != "" ? data.Attributes["sha1"].Value : "")
-						{
+													);
-							machinefound = false;
+												}
-							machinename = "";
+											}
-							description = "";
+										}
-							gameid = 0;
+									}
-						}
+								}
-					}
+							}
-
-					// Process SoftwareList XML-derived DATs
-					else if (format == "softwarelist" && !comment)
-					{
-						if (line.IndexOf("<software ") != -1)
-						{
-							machinefound = true;
-						}
-						else if (line.IndexOf("<description") != -1 && machinefound)
-						{
-							XElement xml = XElement.Parse(line);
-							machinename = xml.Value;
-							gameid = AddGame(sysid, machinename, srcid);
-						}
-						else if (line.IndexOf("<rom") != -1 && machinefound)
-						{
-							AddRom(line, machinename, "rom", gameid, date);
-						}
-						else if (line.IndexOf("<disk") != -1 && machinefound)
-						{
-							AddRom(line, machinename, "disk", gameid, date);
-						}
-						else if (line.IndexOf("</software>") != -1)
-						{
-							machinefound = false;
-							machinename = "";
-							description = "";
-							gameid = 0;
-						}
-					}
-
-					// Process original style RomVault DATs
-					else if (format == "romvault")
-					{
-						if (line.IndexOf("game") != -1 && !machinefound)
-						{
-							machinefound = true;
-						}
-						else if (line.IndexOf("rom (") != -1 && machinefound)
-						{
-							AddRomOld(line, machinename, "rom", gameid, date);
-						}
-						else if (line.IndexOf("disk (") != -1 && machinefound)
-						{
-							AddRomOld(line, machinename, "disk", gameid, date);
-						}
-						else if (line.IndexOf("name \"") != -1 && machinefound)
-						{
-							string machineNamePattern = "^\\s*name \"(.*)\"";
-							Regex machineNameRegex = new Regex(machineNamePattern);
-							Match machineNameMatch = machineNameRegex.Match(line);
-							machinename = machineNameMatch.Groups[1].Value;
-
-							gameid = AddGame(sysid, machinename, srcid);
-						}
-						else if (line.IndexOf("description \"") == -1 && line.IndexOf(")") != -1)
-						{
-							machinefound = false;
-							machinename = "";
-							description = "";
-							gameid = 0;
 						}
 					}
+					node = node.NextSibling;
 				}
 
 				sr.Close();
@@ -464,57 +403,7 @@ namespace DATabase
 			return gameid;
 		}
 
-		private bool AddRom(string line, string machinename, string romtype, long gameid, string date)
+		private bool AddRomHelper(string romtype, long gameid, string name, string date, int size, string crc, string md5, string sha1)
-		{
-			XElement xml = XElement.Parse(line);
-
-			string name = (xml.Attribute("name") != null ? xml.Attribute("name").Value : "");
-			int size = (xml.Attribute("size") != null ? Int32.Parse(xml.Attribute("size").Value) : -1);
-			string crc = (xml.Attribute("crc") != null ? xml.Attribute("crc").Value : "");
-			string md5 = (xml.Attribute("md5") != null ? xml.Attribute("md5").Value : "");
-			string sha1 = (xml.Attribute("sha1") != null ? xml.Attribute("sha1").Value : "");
-
-			return AddRomHelper(machinename, romtype, gameid, name, date, size, crc, md5, sha1);
-		}
-
-		private bool AddRomOld(string line, string machinename, string romtype, long gameid, string date)
-		{
-			string infoPattern = "name \"(.*)\"";
-			Regex infoRegex = new Regex(infoPattern);
-			Match infoMatch = infoRegex.Match(line);
-			string name = infoMatch.Groups[1].Value;
-
-			string[] rominfo = line.Split(' ');
-			int size = -1;
-			string crc = "";
-			string md5 = "";
-			string sha1 = "";
-
-			string next = "";
-			foreach (string info in rominfo)
-			{
-				if (info == "size" || info == "crc" || info == "md5" || info == "sha1")
-				{
-					next = info;
-				}
-				else if (next != "")
-				{
-					switch (next)
-					{
-						case "size": size = Int32.Parse(info); break;
-						case "crc": crc = info; break;
-						case "md5": md5 = info; break;
-						case "sha1": sha1 = info; break;
-						default: break;
-					}
-					next = "";
-				}
-			}
-
-			return AddRomHelper(machinename, romtype, gameid, name, date, size, crc, md5, sha1);
-		}
-
-		private bool AddRomHelper(string machinename, string romtype, long gameid, string name, string date, int size, string crc, string md5, string sha1)
 		{
 			// WOD origninally stripped out any subdirs from the imported files, we do the same
 			name = Path.GetFileName(name);

DATabase/Program.cs

@@ -3,6 +3,8 @@ using System.Collections.Generic;
 using System.Data.SQLite;
 using System.IO;
 
+using System.Xml.Linq;
+
 namespace DATabase
 {
 	class Program
@@ -158,6 +160,36 @@ ORDER BY systems.manufacturer, systems.system";
 
 					break;
 
+				case "-c":
+				case "--convert":
+					// Check if there are enough arguments
+					if (args.Length > 1)
+					{
+						// Check to see if the second argument is a file that exists
+						if (args.Length > 1 && File.Exists(args[1]))
+						{
+							Console.WriteLine("Converting " + args[1]);
+							XElement conv = Converters.RomVaultToXML(File.ReadAllLines(args[1]));
+							FileStream fs = File.OpenWrite(Path.GetFileNameWithoutExtension(args[1]) + ".new.xml");
+							StreamWriter sw = new StreamWriter(fs);
+							sw.Write(conv);
+							sw.Close();
+							fs.Close();
+							Console.WriteLine("Converted file: " + Path.GetFileNameWithoutExtension(args[1]) + ".new.xml");
+						}
+						// If it's invalid, show the help
+						else
+						{
+							Help();
+						}
+					}
+					// If there aren't enough arguments
+					else
+					{
+						Help();
+					}
+					break;
+
 				// Invalid argument
 				default:
 					Help();
@@ -264,9 +296,10 @@ Usage: DATabase <option> (<filename>|<dirname>) | (system=sy) (source=so)
 	-g, --generate	Start tool in generate mode
 	-lso		List all sources (id <= name)
 	-lsy		List all systems (id <= name)
+	-c, --convert	Convert a RV DAT to XML
 
-If started in import mode, either a filename or directory
+If started in import or convert mode, either a filename
-name is required  in order to run.
+or directory name is required  in order to run.
 
 If started in generate mode, here are the possible states:
 	system blank,	source blank	Create MEGAMERGED