claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[RombaSharp] Implement database based directly on Romba KV implementation

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-14 14:04:15 -07:00
parent 756e5bf4d7
commit cbbab3518c
3 changed files with 146 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
RombaSharp/Partials/RombaSharp_Helpers.cs
View File

@@ -232,43 +232,23 @@ namespace SabreTools
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Total uncompressed size
string query = "SELECT SUM(size) FROM data";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Uncompressed size: " + Style.GetBytesReadable((long)slc.ExecuteScalar()));
// Total number of files
query = "SELECT COUNT(*) FROM data";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files: " + (long)slc.ExecuteScalar());
// Total number of files that exist
query = "SELECT COUNT(*) FROM data WHERE indepot=1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files in depots: " + (long)slc.ExecuteScalar());
// Total number of files that are missing
query = "SELECT COUNT(*) FROM data WHERE indepot=0";
slc = new SqliteCommand(query, dbc);
_logger.User("Total files missing: " + (long)slc.ExecuteScalar());
// Total number of CRCs
query = "SELECT COUNT(crc) FROM data WHERE NOT crc=\"null\"";
slc = new SqliteCommand(query, dbc);
string query = "SELECT COUNT(*) FROM crc";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Total CRCs: " + (long)slc.ExecuteScalar());
// Total number of MD5s
query = "SELECT COUNT(md5) FROM data WHERE NOT md5=\"null\"";
query = "SELECT COUNT(*) FROM md5";
slc = new SqliteCommand(query, dbc);
_logger.User("Total MD5s: " + (long)slc.ExecuteScalar());
// Total number of SHA1s
query = "SELECT COUNT(sha1) FROM data WHERE NOT sha1=\"null\"";
query = "SELECT COUNT(*) FROM sha1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total SHA1s: " + (long)slc.ExecuteScalar());
// Total number of DATs
query = "SELECT COUNT(*) FROM dats GROUP BY hash";
query = "SELECT COUNT(*) FROM dat";
slc = new SqliteCommand(query, dbc);
_logger.User("Total DATs: " + (long)slc.ExecuteScalar());
@@ -298,6 +278,7 @@ namespace SabreTools
/// <summary>
/// Export the current database to CSV
/// </summary>
/// <remarks>REDO</remarks>
private static void ExportDatabase()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
@@ -350,7 +331,7 @@ namespace SabreTools
/// <summary>
/// Populate or refresh the database information
/// </summary>
/// <remarks>Each hash has the following attributes: size, crc, md5, sha-1, dathash, indepot</remarks>
/// <remarks>This has no link between Dat and file at all...</remarks>
private static void RefreshDatabase()
{
// Make sure the db is set
@@ -388,7 +369,7 @@ namespace SabreTools
_logger.User("Populating the list of existing DATs");
DateTime start = DateTime.Now;
string query = "SELECT DISTINCT hash FROM dats";
string query = "SELECT DISTINCT hash FROM dat";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -444,6 +425,12 @@ namespace SabreTools
// If the Dat wasn't empty, add the information
if (tempdat.Files.Count != 0)
{
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Loop through the parsed entries
foreach (string romkey in tempdat.Files.Keys)
{
@@ -451,60 +438,55 @@ namespace SabreTools
{
_logger.Verbose("Checking and adding file '" + rom.Name);
query = "SELECT id FROM data WHERE size=" + rom.Size + " AND ("
+ "(crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\"))";
slc = new SqliteCommand(query, dbc);
sldr = slc.ExecuteReader();
// If the hash exists in the database, add the dat hash for that id if needed
if (sldr.HasRows)
if (!String.IsNullOrEmpty(rom.CRC))
{
sldr.Read();
long id = sldr.GetInt64(0);
string squery = "SELECT * FROM dats WHERE id=" + id;
SqliteCommand sslc = new SqliteCommand(squery, dbc);
SqliteDataReader ssldr = sslc.ExecuteReader();
// If the hash doesn't already exist, add it
if (!ssldr.HasRows)
crcquery += " (\"" + rom.CRC + "\"),";
}
if (!String.IsNullOrEmpty(rom.MD5))
{
squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
md5query += " (\"" + rom.MD5 + "\"),";
}
ssldr.Dispose();
sslc.Dispose();
}
// If it doesn't exist, add the hash and the dat hash for a new id
else
if (!String.IsNullOrEmpty(rom.SHA1))
{
string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+ rom.Size + ","
+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+ "0)";
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
long id = -1;
squery = @"select last_insert_rowid()";
sslc = new SqliteCommand(squery, dbc);
id = (long)sslc.ExecuteScalar();
squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
sha1query += " (\"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrEmpty(rom.CRC) && !String.IsNullOrEmpty(rom.SHA1))
{
crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrEmpty(rom.MD5) && !String.IsNullOrEmpty(rom.SHA1))
{
md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),";
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
}
}
_logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));

127
RombaSharp/Partials/RombaSharp_Inits.cs
View File

@@ -47,50 +47,54 @@ namespace SabreTools
List<DatItem> datItems = df.Files[key];
foreach (Rom rom in datItems)
{
string query = "SELECT id FROM data WHERE size=" + rom.Size
+ " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")"
+ " AND indepot=0";
bool add = false;
if (onlyNeeded)
{
string query = @"
SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1
JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1
WHERE crcsha1.crc=""" + rom.CRC + @"""
OR md5sha1.md5=""" + rom.MD5 + @"""
OR md5sha1.sha1=""" + rom.SHA1 + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
// If a row is returned, add the file and change the existence
if (sldr.HasRows)
{
sldr.Read();
long id = sldr.GetInt64(0);
string squery = "UPDATE data SET indepot=1 WHERE id=" + id;
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
// Add the rom to the files that need to be rebuilt
if (need.Files.ContainsKey(key))
{
need.Files[key].Add(rom);
add = true;
}
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
need.Files.Add(key, temp);
}
add = true;
}
// If it doesn't exist, and we're not adding only needed files
else if (!onlyNeeded)
// If we're supposed to add the file to the depot, add it to the list
if (add)
{
string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+ rom.Size + ","
+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+ "1)";
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
_logger.User("Adding file \"" + rom.Name + "\" to the database");
// Insert new or updated information into the database
string query = "INSERT OR IGNORE INTO crc (crc) VALUES (\"" + rom.CRC + "\")";
SqliteCommand slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO md5 (md5) VALUES (\"" + rom.MD5 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES (\"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
// Add the rom to the files that need to be rebuilt
if (need.Files.ContainsKey(key))
@@ -148,58 +152,13 @@ namespace SabreTools
Directory.CreateDirectory(outputFolder);
}
// Then get all hashes associated with this DAT
string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string sha1 = sldr.GetString(0);
string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz");
// Get the depots that are online
List<string> onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList();
// Find the first depot that contains the folder
foreach (string depot in _depots.Keys)
{
// If the depot is online, check it
if (_depots[depot].Item2)
{
if (File.Exists(Path.Combine(depot, filename)))
{
if (copy)
{
if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename))))
{
Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename)));
}
try
{
File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true);
}
catch { }
}
else
{
ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger);
}
continue;
}
}
}
}
}
// Now that we have extracted everything, we rebuild to the output folder
if (!copy)
{
List<string> temp = new List<string>();
temp.Add(_tmpdir);
SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, true, false, false, asl, false, _logger);
// Loop over the depots and rebuild as needed
SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, copy, copy, asl, false, _logger);
ss.StartProcessing();
}
}
dbc.Dispose();
}
@@ -305,7 +264,7 @@ namespace SabreTools
// Now, search for each of them and return true or false for each
foreach (string input in crc)
{
string query = "SELECT * FROM data WHERE crc=\"" + input + "\"";
string query = "SELECT * FROM crc WHERE crc=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -322,7 +281,7 @@ namespace SabreTools
}
foreach (string input in md5)
{
string query = "SELECT * FROM data WHERE md5=\"" + input + "\"";
string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -339,7 +298,7 @@ namespace SabreTools
}
foreach (string input in sha1)
{
string query = "SELECT * FROM data WHERE sha1=\"" + input + "\"";
string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)

48
SabreTools.Helper/Tools/DatabaseTools.cs
View File

@@ -72,23 +72,51 @@ namespace SabreTools.Helper
if (type == "rombasharp")
{
string query = @"
CREATE TABLE IF NOT EXISTS data (
'id' INTEGER NOT NULL,
'size' INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS crc (
'crc' TEXT NOT NULL,
'md5' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
'indepot' INTEGER NOT NULL,
PRIMARY KEY (id)
PRIMARY KEY (crc)
)";
SqliteCommand slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS dats (
'id' INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS md5 (
'md5' TEXT NOT NULL,
PRIMARY KEY (md5)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS sha1 (
'sha1' TEXT NOT NULL,
PRIMARY KEY (sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS crcsha1 (
'crc' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
PRIMARY KEY (crc, sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS md5sha1 (
'md5' TEXT NOT NULL,
'sha1' TEXT NOT NULL,
PRIMARY KEY (md5, sha1)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
query = @"
CREATE TABLE IF NOT EXISTS dat (
'hash' TEXT NOT NULL,
PRIMARY KEY (id, hash)
PRIMARY KEY (hash)
)";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();