claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add stage 1 changes for better merging

Browse Source 
The new merging process is both parse and merge in one step using an in-memory database. The next steps are to get the data in this database and write it out properly.
This commit is contained in:
Matt Nadareski
2016-04-27 00:53:15 -07:00
parent 6e6a2f9391
commit bce1cc5839
3 changed files with 222 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
DATabase/MergeDiff.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using Mono.Data.Sqlite;
using System.IO;
using SabreTools.Helper;
@@ -105,9 +106,13 @@ namespace SabreTools
List<RomData> A = new List<RomData>();
SqliteConnection dbc = DBTools.InMemoryDb();
foreach (string input in _inputs)
{
_logger.Log("Adding DAT: " + input);
RomManipulation.Parse2(input, 0, 0, _dedup, dbc, _logger);
List<RomData> B = RomManipulation.Parse(input, 0, 0, _logger);
if (_diff)
{
@@ -119,6 +124,13 @@ namespace SabreTools
}
}
// Until I find a way to output the roms from the db, here's just a count of the items in it
using (SqliteCommand slc = new SqliteCommand("SELECT count(*) FROM roms", dbc))
{
_logger.Log("Total number of lines in database: " + slc.ExecuteScalar());
}
dbc.Close();
// If we're in Alldiff mode, we can only use the first 2 inputs
if (_ad)
{

29
SabreHelper/DBTools.cs
View File

@@ -177,6 +177,35 @@ CREATE TABLE IF NOT EXISTS gamesource (
}
}
/// <summary>
/// Create an in-memory database table for import and merging
/// </summary>
/// <returns>An active database connection</returns>
public static SqliteConnection InMemoryDb()
{
SqliteConnection dbc = new SqliteConnection("Data Source=:memory:;Version = 3;");
dbc.Open();
string query = @"
CREATE TABLE IF NOT EXISTS roms (
'id' INTEGER PRIMARY KEY NOT NULL,
'game' TEXT NOT NULL,
'name' TEXT NOT NULL,
'type' TEXT NOT NULL DEFAULT 'rom',
'sysid' INTEGER NOT NULL,
'srcid' INTEGER NOT NULL,
'size' INTEGER NOT NULL DEFAULT -1,
'crc' TEXT NOT NULL,
'md5' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
'dupe' TEXT NOT NULL DEFAULT 'false'
)";
SqliteCommand slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
return dbc;
}
/// <summary>
/// Add a new source to the database if it doesn't already exist
/// </summary>

181
SabreHelper/RomManipulation.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using Mono.Data.Sqlite;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
@@ -344,6 +345,185 @@ namespace SabreTools.Helper
return roms;
}
/// <summary>
/// Parse a DAT and return all found games and roms within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="sysid">System ID for the DAT</param>
/// <param name="srcid">Source ID for the DAT</param>
/// <param name="merge">True if files should be matched by hash alone, false otherwise</param>
/// <param name="dbc">Database connection for adding found ROMs</param>
/// <param name="logger">Logger object for console and/or file output</param>
/// <returns>True if no errors occur, false otherwise</returns>
public static bool Parse2(string filename, int sysid, int srcid, bool merge, SqliteConnection dbc, Logger logger)
{
XmlTextReader xtr = GetXmlTextReader(filename, logger);
xtr.WhitespaceHandling = WhitespaceHandling.None;
bool superdat = false, shouldbreak = false;
string parent = "";
// If the reader is null, return false
if (xtr == null)
{
return false;
}
xtr.MoveToContent();
while (xtr.NodeType != XmlNodeType.None)
{
// We only want elements
if (xtr.NodeType != XmlNodeType.Element)
{
xtr.Read();
continue;
}
switch (xtr.Name)
{
case "datafile":
case "softwarelist":
parent = xtr.Name;
xtr.Read();
break;
case "header":
xtr.ReadToDescendant("name");
superdat = (xtr.ReadElementContentAsString() != null ? xtr.ReadElementContentAsString().Contains(" - SuperDAT") : false);
while (xtr.Name != "header")
{
xtr.Read();
}
xtr.Read();
break;
case "machine":
case "game":
case "software":
string temptype = xtr.Name;
string tempname = "";
// We want to process the entire subtree of the game
XmlReader subreader = xtr.ReadSubtree();
if (subreader != null)
{
if (temptype == "software" && subreader.ReadToFollowing("description"))
{
tempname = subreader.ReadElementContentAsString();
}
else
{
// There are rare cases where a malformed XML will not have the required attributes. We can only skip them.
if (xtr.AttributeCount == 0)
{
logger.Error("No attributes were found");
xtr.ReadToNextSibling(xtr.Name);
continue;
}
tempname = xtr.GetAttribute("name");
}
if (superdat)
{
tempname = Regex.Match(tempname, @".*?\\(.*)").Groups[1].Value;
}
while (subreader.Read())
{
// We only want elements
if (subreader.NodeType != XmlNodeType.Element)
{
continue;
}
// Get the roms from the machine
switch (subreader.Name)
{
case "rom":
case "disk":
// Take care of hex-sized files
long size = -1;
if (xtr.GetAttribute("size") != null && xtr.GetAttribute("size").Contains("0x"))
{
size = Convert.ToInt64(xtr.GetAttribute("size"), 16);
}
else if (xtr.GetAttribute("size") != null)
{
Int64.TryParse(xtr.GetAttribute("size"), out size);
}
// If the rom doesn't exist, add it to the database
string query = @"SELECT sha1 FROM roms WHERE size=" + size +
(xtr.GetAttribute("crc") != null ? " AND crc='" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : "") +
(xtr.GetAttribute("md5") != null ? " AND md5='" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : "") +
(xtr.GetAttribute("sha1") != null ? " AND sha1='" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : "") +
(merge ? "" : " AND game='" + tempname.Replace("'", "''") + "' AND name='" + xtr.GetAttribute("name").Replace("'", "''") + "' AND sysid=" + sysid + " AND srcid=" + srcid);
using (SqliteCommand slc = new SqliteCommand(query, dbc))
{
using (SqliteDataReader sldr = slc.ExecuteReader())
{
// If there's no returns, then add the file
if (!sldr.HasRows)
{
query = @"INSERT INTO roms
(game, name, type, sysid, srcid, size, crc, md5, sha1)
VALUES ('" + tempname.Replace("'", "''") + "', '" +
xtr.GetAttribute("name").Replace("'", "''") + "', '" +
xtr.Name + "', " +
sysid + ", " +
srcid + ", " +
size +
(xtr.GetAttribute("crc") != null ? ", '" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : ", ''") +
(xtr.GetAttribute("md5") != null ? ", '" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : ", ''") +
(xtr.GetAttribute("sha1") != null ? ", '" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : ", ''") +
")";
using (SqliteCommand sslc = new SqliteCommand(query, dbc))
{
sslc.ExecuteNonQuery();
}
}
// Otherwise, set the dupe flag to true
else
{
query = @"UPDATE roms SET dupe='true' WHERE size=" + size +
(xtr.GetAttribute("crc") != null ? " AND crc='" + xtr.GetAttribute("crc").ToLowerInvariant().Trim() + "'" : "") +
(xtr.GetAttribute("md5") != null ? " AND md5='" + xtr.GetAttribute("md5").ToLowerInvariant().Trim() + "'" : "") +
(xtr.GetAttribute("sha1") != null ? " AND sha1='" + xtr.GetAttribute("sha1").ToLowerInvariant().Trim() + "'" : "") +
(merge ? "" : " AND game='" + tempname.Replace("'", "''") + "' AND name='" + xtr.GetAttribute("name").Replace("'", "''") + "' AND sysid=" + sysid + " AND srcid=" + srcid);
using (SqliteCommand sslc = new SqliteCommand(query, dbc))
{
sslc.ExecuteNonQuery();
}
}
}
}
break;
}
}
}
// Read to next game
if (!xtr.ReadToNextSibling(temptype))
{
shouldbreak = true;
}
break;
default:
xtr.Read();
break;
}
// If we hit an endpoint, break out of the loop early
if (shouldbreak)
{
break;
}
}
return true;
}
/// <summary>
/// Merge an arbitrary set of ROMs based on the supplied information
/// </summary>
@@ -436,6 +616,7 @@ namespace SabreTools.Helper
return outroms;
}
/// <summary>
/// Sort a list of RomData objects by SystemID, SourceID, Game, and Name (in order)
/// </summary>