claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[RombaSharp] Implement database based directly on Romba KV implementation

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-14 14:04:15 -07:00
parent 756e5bf4d7
commit cbbab3518c
3 changed files with 146 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
RombaSharp/Partials/RombaSharp_Helpers.cs
View File

@@ -232,43 +232,23 @@ namespace SabreTools
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Total uncompressed size
string query = "SELECT SUM(size) FROM data";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Uncompressed size: " + Style.GetBytesReadable((long)slc.ExecuteScalar()));
// Total number of files
query = "SELECT COUNT(*) FROM data";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files: " + (long)slc.ExecuteScalar());
// Total number of files that exist
query = "SELECT COUNT(*) FROM data WHERE indepot=1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files in depots: " + (long)slc.ExecuteScalar());
// Total number of files that are missing
query = "SELECT COUNT(*) FROM data WHERE indepot=0";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files missing: " + (long)slc.ExecuteScalar());
// Total number of CRCs
query = "SELECT COUNT(crc) FROM data WHERE NOT crc=\"null\"";
slc = new SqliteCommand(query, dbc);
string query = "SELECT COUNT(*) FROM crc";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Total CRCs: " + (long)slc.ExecuteScalar());
// Total number of MD5s
query = "SELECT COUNT(md5) FROM data WHERE NOT md5=\"null\"";
query = "SELECT COUNT(*) FROM md5";
slc = new SqliteCommand(query, dbc);
_logger.User("Total MD5s: " + (long)slc.ExecuteScalar());
// Total number of SHA1s
query = "SELECT COUNT(sha1) FROM data WHERE NOT sha1=\"null\"";
query = "SELECT COUNT(*) FROM sha1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total SHA1s: " + (long)slc.ExecuteScalar());
// Total number of DATs
query = "SELECT COUNT(*) FROM dats GROUP BY hash";
query = "SELECT COUNT(*) FROM dat";
slc = new SqliteCommand(query, dbc);
_logger.User("Total DATs: " + (long)slc.ExecuteScalar());
@@ -298,6 +278,7 @@ namespace SabreTools
/// <summary>
/// Export the current database to CSV
/// </summary>
/// <remarks>REDO</remarks>
private static void ExportDatabase()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
@@ -350,7 +331,7 @@ namespace SabreTools
/// <summary>
/// Populate or refresh the database information
/// </summary>
/// <remarks>Each hash has the following attributes: size, crc, md5, sha-1, dathash, indepot</remarks>
/// <remarks>This has no link between Dat and file at all...</remarks>
private static void RefreshDatabase()
{
// Make sure the db is set
@@ -388,7 +369,7 @@ namespace SabreTools
_logger.User("Populating the list of existing DATs");
DateTime start = DateTime.Now;
string query = "SELECT DISTINCT hash FROM dats";
string query = "SELECT DISTINCT hash FROM dat";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -444,6 +425,12 @@ namespace SabreTools
// If the Dat wasn't empty, add the information
if (tempdat.Files.Count != 0)
{
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Loop through the parsed entries
foreach (string romkey in tempdat.Files.Keys)
{
@@ -451,60 +438,55 @@ namespace SabreTools
{
_logger.Verbose("Checking and adding file '" + rom.Name);
query = "SELECT id FROM data WHERE size=" + rom.Size + " AND ("
+ "(crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\"))";
slc = new SqliteCommand(query, dbc);
sldr = slc.ExecuteReader();
// If the hash exists in the database, add the dat hash for that id if needed
if (sldr.HasRows)
if (!String.IsNullOrEmpty(rom.CRC))
{
sldr.Read();
long id = sldr.GetInt64(0);
string squery = "SELECT * FROM dats WHERE id=" + id;
SqliteCommand sslc = new SqliteCommand(squery, dbc);
SqliteDataReader ssldr = sslc.ExecuteReader();
// If the hash doesn't already exist, add it
if (!ssldr.HasRows)
{
squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
}
ssldr.Dispose();
sslc.Dispose();
crcquery += " (\"" + rom.CRC + "\"),";
}
// If it doesn't exist, add the hash and the dat hash for a new id
else
if (!String.IsNullOrEmpty(rom.MD5))
{
string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+ rom.Size + ","
+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+ "0)";
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
long id = -1;
squery = @"select last_insert_rowid()";
sslc = new SqliteCommand(squery, dbc);
id = (long)sslc.ExecuteScalar();
squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
md5query += " (\"" + rom.MD5 + "\"),";
}
if (!String.IsNullOrEmpty(rom.SHA1))
{
sha1query += " (\"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrEmpty(rom.CRC) && !String.IsNullOrEmpty(rom.SHA1))
{
crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrEmpty(rom.MD5) && !String.IsNullOrEmpty(rom.SHA1))
{
md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),";
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
}
}
_logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));

141
RombaSharp/Partials/RombaSharp_Inits.cs
View File

@@ -47,50 +47,54 @@ namespace SabreTools
List<DatItem> datItems = df.Files[key];
foreach (Rom rom in datItems)
{
string query = "SELECT id FROM data WHERE size=" + rom.Size
+ " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")"
+ " AND indepot=0";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
// If a row is returned, add the file and change the existence
if (sldr.HasRows)
bool add = false;
if (onlyNeeded)
{
sldr.Read();
long id = sldr.GetInt64(0);
string query = @"
SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1
JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1
WHERE crcsha1.crc=""" + rom.CRC + @"""
OR md5sha1.md5=""" + rom.MD5 + @"""
OR md5sha1.sha1=""" + rom.SHA1 + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
string squery = "UPDATE data SET indepot=1 WHERE id=" + id;
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
// Add the rom to the files that need to be rebuilt
if (need.Files.ContainsKey(key))
// If a row is returned, add the file and change the existence
if (sldr.HasRows)
{
need.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
need.Files.Add(key, temp);
add = true;
}
}
// If it doesn't exist, and we're not adding only needed files
else if (!onlyNeeded)
else
{
string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+ rom.Size + ","
+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+ "1)";
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
add = true;
}
// If we're supposed to add the file to the depot, add it to the list
if (add)
{
_logger.User("Adding file \"" + rom.Name + "\" to the database");
// Insert new or updated information into the database
string query = "INSERT OR IGNORE INTO crc (crc) VALUES (\"" + rom.CRC + "\")";
SqliteCommand slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO md5 (md5) VALUES (\"" + rom.MD5 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES (\"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
// Add the rom to the files that need to be rebuilt
if (need.Files.ContainsKey(key))
@@ -148,57 +152,12 @@ namespace SabreTools
Directory.CreateDirectory(outputFolder);
}
// Then get all hashes associated with this DAT
string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string sha1 = sldr.GetString(0);
string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz");
// Get the depots that are online
List<string> onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList();
// Find the first depot that contains the folder
foreach (string depot in _depots.Keys)
{
// If the depot is online, check it
if (_depots[depot].Item2)
{
if (File.Exists(Path.Combine(depot, filename)))
{
if (copy)
{
if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename))))
{
Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename)));
}
try
{
File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true);
}
catch { }
}
else
{
ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger);
}
continue;
}
}
}
}
}
// Now that we have extracted everything, we rebuild to the output folder
if (!copy)
{
List<string> temp = new List<string>();
temp.Add(_tmpdir);
SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, true, false, false, asl, false, _logger);
ss.StartProcessing();
}
// Loop over the depots and rebuild as needed
SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, copy, copy, asl, false, _logger);
ss.StartProcessing();
}
dbc.Dispose();
@@ -305,7 +264,7 @@ namespace SabreTools
// Now, search for each of them and return true or false for each
foreach (string input in crc)
{
string query = "SELECT * FROM data WHERE crc=\"" + input + "\"";
string query = "SELECT * FROM crc WHERE crc=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -322,7 +281,7 @@ namespace SabreTools
}
foreach (string input in md5)
{
string query = "SELECT * FROM data WHERE md5=\"" + input + "\"";
string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -339,7 +298,7 @@ namespace SabreTools
}
foreach (string input in sha1)
{
string query = "SELECT * FROM data WHERE sha1=\"" + input + "\"";
string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)

50
SabreTools.Helper/Tools/DatabaseTools.cs
View File

@@ -72,23 +72,51 @@ namespace SabreTools.Helper
if (type == "rombasharp")
{
string query = @"
CREATE TABLE IF NOT EXISTS data (
'id' INTEGER NOT NULL,
'size' INTEGER NOT NULL,
'crc' TEXT NOT NULL,
'md5' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
'indepot' INTEGER NOT NULL,
PRIMARY KEY (id)
CREATE TABLE IF NOT EXISTS crc (
'crc' TEXT NOT NULL,
PRIMARY KEY (crc)
)";
SqliteCommand slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS dats (
'id' INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS md5 (
'md5' TEXT NOT NULL,
PRIMARY KEY (md5)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS sha1 (
'sha1' TEXT NOT NULL,
PRIMARY KEY (sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS crcsha1 (
'crc' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
PRIMARY KEY (crc, sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS md5sha1 (
'md5' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
PRIMARY KEY (md5, sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS dat (
'hash' TEXT NOT NULL,
PRIMARY KEY (id, hash)
PRIMARY KEY (hash)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();