diff --git a/DATabase/MergeDiff.cs b/DATabase/MergeDiff.cs index f150c5ac..e8aa3a3a 100644 --- a/DATabase/MergeDiff.cs +++ b/DATabase/MergeDiff.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using Mono.Data.Sqlite; using System.IO; using SabreTools.Helper; @@ -105,9 +106,13 @@ namespace SabreTools List A = new List(); + SqliteConnection dbc = DBTools.InMemoryDb(); + foreach (string input in _inputs) { _logger.Log("Adding DAT: " + input); + RomManipulation.Parse2(input, 0, 0, _dedup, dbc, _logger); + List B = RomManipulation.Parse(input, 0, 0, _logger); if (_diff) { @@ -119,6 +124,13 @@ namespace SabreTools } } + // Until I find a way to output the roms from the db, here's just a count of the items in it + using (SqliteCommand slc = new SqliteCommand("SELECT count(*) FROM roms", dbc)) + { + _logger.Log("Total number of lines in database: " + slc.ExecuteScalar()); + } + dbc.Close(); + // If we're in Alldiff mode, we can only use the first 2 inputs if (_ad) { diff --git a/SabreHelper/DBTools.cs b/SabreHelper/DBTools.cs index a01ce247..4d130486 100644 --- a/SabreHelper/DBTools.cs +++ b/SabreHelper/DBTools.cs @@ -177,6 +177,35 @@ CREATE TABLE IF NOT EXISTS gamesource ( } } + /// + /// Create an in-memory database table for import and merging + /// + /// An active database connection + public static SqliteConnection InMemoryDb() + { + SqliteConnection dbc = new SqliteConnection("Data Source=:memory:;Version = 3;"); + dbc.Open(); + + string query = @" +CREATE TABLE IF NOT EXISTS roms ( + 'id' INTEGER PRIMARY KEY NOT NULL, + 'game' TEXT NOT NULL, + 'name' TEXT NOT NULL, + 'type' TEXT NOT NULL DEFAULT 'rom', + 'sysid' INTEGER NOT NULL, + 'srcid' INTEGER NOT NULL, + 'size' INTEGER NOT NULL DEFAULT -1, + 'crc' TEXT NOT NULL, + 'md5' TEXT NOT NULL, + 'sha1' TEXT NOT NULL, + 'dupe' TEXT NOT NULL DEFAULT 'false' +)"; + SqliteCommand slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + return dbc; + } + /// /// Add a new source to the database if it doesn't already exist /// diff --git a/SabreHelper/RomManipulation.cs b/SabreHelper/RomManipulation.cs index e5aefc9c..9c47f69e 100644 --- a/SabreHelper/RomManipulation.cs +++ b/SabreHelper/RomManipulation.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using Mono.Data.Sqlite; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -344,6 +345,185 @@ namespace SabreTools.Helper return roms; } + /// + /// Parse a DAT and return all found games and roms within + /// + /// Name of the file to be parsed + /// System ID for the DAT + /// Source ID for the DAT + /// True if files should be matched by hash alone, false otherwise + /// Database connection for adding found ROMs + /// Logger object for console and/or file output + /// True if no errors occur, false otherwise + public static bool Parse2(string filename, int sysid, int srcid, bool merge, SqliteConnection dbc, Logger logger) + { + XmlTextReader xtr = GetXmlTextReader(filename, logger); + xtr.WhitespaceHandling = WhitespaceHandling.None; + bool superdat = false, shouldbreak = false; + string parent = ""; + + // If the reader is null, return false + if (xtr == null) + { + return false; + } + + xtr.MoveToContent(); + while (xtr.NodeType != XmlNodeType.None) + { + // We only want elements + if (xtr.NodeType != XmlNodeType.Element) + { + xtr.Read(); + continue; + } + + switch (xtr.Name) + { + case "datafile": + case "softwarelist": + parent = xtr.Name; + xtr.Read(); + break; + case "header": + xtr.ReadToDescendant("name"); + superdat = (xtr.ReadElementContentAsString() != null ? xtr.ReadElementContentAsString().Contains(" - SuperDAT") : false); + while (xtr.Name != "header") + { + xtr.Read(); + } + xtr.Read(); + break; + case "machine": + case "game": + case "software": + string temptype = xtr.Name; + string tempname = ""; + + // We want to process the entire subtree of the game + XmlReader subreader = xtr.ReadSubtree(); + + if (subreader != null) + { + if (temptype == "software" && subreader.ReadToFollowing("description")) + { + tempname = subreader.ReadElementContentAsString(); + } + else + { + // There are rare cases where a malformed XML will not have the required attributes. We can only skip them. + if (xtr.AttributeCount == 0) + { + logger.Error("No attributes were found"); + xtr.ReadToNextSibling(xtr.Name); + continue; + } + tempname = xtr.GetAttribute("name"); + } + + if (superdat) + { + tempname = Regex.Match(tempname, @".*?\\(.*)").Groups[1].Value; + } + + while (subreader.Read()) + { + // We only want elements + if (subreader.NodeType != XmlNodeType.Element) + { + continue; + } + + // Get the roms from the machine + switch (subreader.Name) + { + case "rom": + case "disk": + // Take care of hex-sized files + long size = -1; + if (xtr.GetAttribute("size") != null && xtr.GetAttribute("size").Contains("0x")) + { + size = Convert.ToInt64(xtr.GetAttribute("size"), 16); + } + else if (xtr.GetAttribute("size") != null) + { + Int64.TryParse(xtr.GetAttribute("size"), out size); + } + + // If the rom doesn't exist, add it to the database + string query = @"SELECT sha1 FROM roms WHERE size=" + size + +(xtr.GetAttribute("crc") != null ? " AND crc='" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : "") + +(xtr.GetAttribute("md5") != null ? " AND md5='" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : "") + +(xtr.GetAttribute("sha1") != null ? " AND sha1='" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : "") + +(merge ? "" : " AND game='" + tempname.Replace("'", "''") + "' AND name='" + xtr.GetAttribute("name").Replace("'", "''") + "' AND sysid=" + sysid + " AND srcid=" + srcid); + + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + // If there's no returns, then add the file + if (!sldr.HasRows) + { + query = @"INSERT INTO roms +(game, name, type, sysid, srcid, size, crc, md5, sha1) +VALUES ('" + tempname.Replace("'", "''") + "', '" + + xtr.GetAttribute("name").Replace("'", "''") + "', '" + + xtr.Name + "', " + + sysid + ", " + + srcid + ", " + + size + + (xtr.GetAttribute("crc") != null ? ", '" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : ", ''") + + (xtr.GetAttribute("md5") != null ? ", '" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : ", ''") + + (xtr.GetAttribute("sha1") != null ? ", '" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : ", ''") + + ")"; + using (SqliteCommand sslc = new SqliteCommand(query, dbc)) + { + sslc.ExecuteNonQuery(); + } + } + // Otherwise, set the dupe flag to true + else + { + query = @"UPDATE roms SET dupe='true' WHERE size=" + size + +(xtr.GetAttribute("crc") != null ? " AND crc='" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : "") + +(xtr.GetAttribute("md5") != null ? " AND md5='" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : "") + +(xtr.GetAttribute("sha1") != null ? " AND sha1='" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : "") + +(merge ? "" : " AND game='" + tempname.Replace("'", "''") + "' AND name='" + xtr.GetAttribute("name").Replace("'", "''") + "' AND sysid=" + sysid + " AND srcid=" + srcid); + + using (SqliteCommand sslc = new SqliteCommand(query, dbc)) + { + sslc.ExecuteNonQuery(); + } + } + } + } + + break; + } + } + } + + // Read to next game + if (!xtr.ReadToNextSibling(temptype)) + { + shouldbreak = true; + } + break; + default: + xtr.Read(); + break; + } + + // If we hit an endpoint, break out of the loop early + if (shouldbreak) + { + break; + } + } + + return true; + } + /// /// Merge an arbitrary set of ROMs based on the supplied information /// @@ -436,6 +616,7 @@ namespace SabreTools.Helper return outroms; } + /// /// Sort a list of RomData objects by SystemID, SourceID, Game, and Name (in order) ///