diff --git a/RombaSharp/Partials/RombaSharp_Helpers.cs b/RombaSharp/Partials/RombaSharp_Helpers.cs index 0ddfcad7..c430af3b 100644 --- a/RombaSharp/Partials/RombaSharp_Helpers.cs +++ b/RombaSharp/Partials/RombaSharp_Helpers.cs @@ -232,43 +232,23 @@ namespace SabreTools SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); - // Total uncompressed size - string query = "SELECT SUM(size) FROM data"; - SqliteCommand slc = new SqliteCommand(query, dbc); - _logger.User("Uncompressed size: " + Style.GetBytesReadable((long)slc.ExecuteScalar())); - - // Total number of files - query = "SELECT COUNT(*) FROM data"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total files: " + (long)slc.ExecuteScalar()); - - // Total number of files that exist - query = "SELECT COUNT(*) FROM data WHERE indepot=1"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total files in depots: " + (long)slc.ExecuteScalar()); - - // Total number of files that are missing - query = "SELECT COUNT(*) FROM data WHERE indepot=0"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total files missing: " + (long)slc.ExecuteScalar()); - // Total number of CRCs - query = "SELECT COUNT(crc) FROM data WHERE NOT crc=\"null\""; - slc = new SqliteCommand(query, dbc); + string query = "SELECT COUNT(*) FROM crc"; + SqliteCommand slc = new SqliteCommand(query, dbc); _logger.User("Total CRCs: " + (long)slc.ExecuteScalar()); // Total number of MD5s - query = "SELECT COUNT(md5) FROM data WHERE NOT md5=\"null\""; + query = "SELECT COUNT(*) FROM md5"; slc = new SqliteCommand(query, dbc); _logger.User("Total MD5s: " + (long)slc.ExecuteScalar()); // Total number of SHA1s - query = "SELECT COUNT(sha1) FROM data WHERE NOT sha1=\"null\""; + query = "SELECT COUNT(*) FROM sha1"; slc = new SqliteCommand(query, dbc); _logger.User("Total SHA1s: " + (long)slc.ExecuteScalar()); // Total number of DATs - query = "SELECT COUNT(*) FROM dats GROUP BY hash"; + query = "SELECT COUNT(*) FROM dat"; slc = new SqliteCommand(query, dbc); _logger.User("Total DATs: " + (long)slc.ExecuteScalar()); @@ -298,6 +278,7 @@ namespace SabreTools /// /// Export the current database to CSV /// + /// REDO private static void ExportDatabase() { SqliteConnection dbc = new SqliteConnection(_connectionString); @@ -350,7 +331,7 @@ namespace SabreTools /// /// Populate or refresh the database information /// - /// Each hash has the following attributes: size, crc, md5, sha-1, dathash, indepot + /// This has no link between Dat and file at all... private static void RefreshDatabase() { // Make sure the db is set @@ -388,7 +369,7 @@ namespace SabreTools _logger.User("Populating the list of existing DATs"); DateTime start = DateTime.Now; - string query = "SELECT DISTINCT hash FROM dats"; + string query = "SELECT DISTINCT hash FROM dat"; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -444,6 +425,12 @@ namespace SabreTools // If the Dat wasn't empty, add the information if (tempdat.Files.Count != 0) { + string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; + string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; + string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES"; + string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; + string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; + // Loop through the parsed entries foreach (string romkey in tempdat.Files.Keys) { @@ -451,60 +438,55 @@ namespace SabreTools { _logger.Verbose("Checking and adding file '" + rom.Name); - query = "SELECT id FROM data WHERE size=" + rom.Size + " AND (" - + "(crc=\"" + rom.CRC + "\" OR crc=\"null\")" - + " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")" - + " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\"))"; - slc = new SqliteCommand(query, dbc); - sldr = slc.ExecuteReader(); - - // If the hash exists in the database, add the dat hash for that id if needed - if (sldr.HasRows) + if (!String.IsNullOrEmpty(rom.CRC)) { - sldr.Read(); - long id = sldr.GetInt64(0); - - string squery = "SELECT * FROM dats WHERE id=" + id; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - SqliteDataReader ssldr = sslc.ExecuteReader(); - - // If the hash doesn't already exist, add it - if (!ssldr.HasRows) - { - squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")"; - sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - } - - ssldr.Dispose(); - sslc.Dispose(); + crcquery += " (\"" + rom.CRC + "\"),"; } - - // If it doesn't exist, add the hash and the dat hash for a new id - else + if (!String.IsNullOrEmpty(rom.MD5)) { - string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES (" - + rom.Size + "," - + "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," - + "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," - + "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," - + "0)"; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - - long id = -1; - - squery = @"select last_insert_rowid()"; - sslc = new SqliteCommand(squery, dbc); - id = (long)sslc.ExecuteScalar(); - - squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")"; - sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - sslc.Dispose(); + md5query += " (\"" + rom.MD5 + "\"),"; + } + if (!String.IsNullOrEmpty(rom.SHA1)) + { + sha1query += " (\"" + rom.SHA1 + "\"),"; + } + if (!String.IsNullOrEmpty(rom.CRC) && !String.IsNullOrEmpty(rom.SHA1)) + { + crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),"; + } + if (!String.IsNullOrEmpty(rom.MD5) && !String.IsNullOrEmpty(rom.SHA1)) + { + md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),"; } } } + + // Now run the queries after fixing them + if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") + { + slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") + { + slc = new SqliteCommand(md5query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES") + { + slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") + { + slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } + if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") + { + slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + } } } _logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index 4540cbe5..9d8df223 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -47,50 +47,54 @@ namespace SabreTools List datItems = df.Files[key]; foreach (Rom rom in datItems) { - string query = "SELECT id FROM data WHERE size=" + rom.Size - + " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")" - + " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")" - + " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")" - + " AND indepot=0"; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - - // If a row is returned, add the file and change the existence - if (sldr.HasRows) + bool add = false; + if (onlyNeeded) { - sldr.Read(); - long id = sldr.GetInt64(0); + string query = @" +SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1 + JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1 +WHERE crcsha1.crc=""" + rom.CRC + @""" + OR md5sha1.md5=""" + rom.MD5 + @""" + OR md5sha1.sha1=""" + rom.SHA1 + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); - string squery = "UPDATE data SET indepot=1 WHERE id=" + id; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - sslc.Dispose(); - - // Add the rom to the files that need to be rebuilt - if (need.Files.ContainsKey(key)) + // If a row is returned, add the file and change the existence + if (sldr.HasRows) { - need.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - need.Files.Add(key, temp); + add = true; } } - - // If it doesn't exist, and we're not adding only needed files - else if (!onlyNeeded) + else { - string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES (" - + rom.Size + "," - + "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," - + "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," - + "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," - + "1)"; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - sslc.Dispose(); + add = true; + } + + // If we're supposed to add the file to the depot, add it to the list + if (add) + { + _logger.User("Adding file \"" + rom.Name + "\" to the database"); + + // Insert new or updated information into the database + string query = "INSERT OR IGNORE INTO crc (crc) VALUES (\"" + rom.CRC + "\")"; + SqliteCommand slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = "INSERT OR IGNORE INTO md5 (md5) VALUES (\"" + rom.MD5 + "\")"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES (\"" + rom.SHA1 + "\")"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\")"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\")"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); // Add the rom to the files that need to be rebuilt if (need.Files.ContainsKey(key)) @@ -148,57 +152,12 @@ namespace SabreTools Directory.CreateDirectory(outputFolder); } - // Then get all hashes associated with this DAT - string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\""; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - if (sldr.HasRows) - { - while (sldr.Read()) - { - string sha1 = sldr.GetString(0); - string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz"); + // Get the depots that are online + List onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList(); - // Find the first depot that contains the folder - foreach (string depot in _depots.Keys) - { - // If the depot is online, check it - if (_depots[depot].Item2) - { - if (File.Exists(Path.Combine(depot, filename))) - { - if (copy) - { - if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename)))) - { - Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename))); - } - - try - { - File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true); - } - catch { } - } - else - { - ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger); - } - continue; - } - } - } - } - } - - // Now that we have extracted everything, we rebuild to the output folder - if (!copy) - { - List temp = new List(); - temp.Add(_tmpdir); - SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, true, false, false, asl, false, _logger); - ss.StartProcessing(); - } + // Loop over the depots and rebuild as needed + SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, copy, copy, asl, false, _logger); + ss.StartProcessing(); } dbc.Dispose(); @@ -305,7 +264,7 @@ namespace SabreTools // Now, search for each of them and return true or false for each foreach (string input in crc) { - string query = "SELECT * FROM data WHERE crc=\"" + input + "\""; + string query = "SELECT * FROM crc WHERE crc=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -322,7 +281,7 @@ namespace SabreTools } foreach (string input in md5) { - string query = "SELECT * FROM data WHERE md5=\"" + input + "\""; + string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -339,7 +298,7 @@ namespace SabreTools } foreach (string input in sha1) { - string query = "SELECT * FROM data WHERE sha1=\"" + input + "\""; + string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) diff --git a/SabreTools.Helper/Tools/DatabaseTools.cs b/SabreTools.Helper/Tools/DatabaseTools.cs index 7f780fd1..3f9757f0 100644 --- a/SabreTools.Helper/Tools/DatabaseTools.cs +++ b/SabreTools.Helper/Tools/DatabaseTools.cs @@ -72,23 +72,51 @@ namespace SabreTools.Helper if (type == "rombasharp") { string query = @" -CREATE TABLE IF NOT EXISTS data ( - 'id' INTEGER NOT NULL, - 'size' INTEGER NOT NULL, - 'crc' TEXT NOT NULL, - 'md5' TEXT NOT NULL, - 'sha1' TEXT NOT NULL, - 'indepot' INTEGER NOT NULL, - PRIMARY KEY (id) +CREATE TABLE IF NOT EXISTS crc ( + 'crc' TEXT NOT NULL, + PRIMARY KEY (crc) )"; SqliteCommand slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); query = @" -CREATE TABLE IF NOT EXISTS dats ( - 'id' INTEGER NOT NULL, +CREATE TABLE IF NOT EXISTS md5 ( + 'md5' TEXT NOT NULL, + PRIMARY KEY (md5) +)"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = @" +CREATE TABLE IF NOT EXISTS sha1 ( + 'sha1' TEXT NOT NULL, + PRIMARY KEY (sha1) +)"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = @" +CREATE TABLE IF NOT EXISTS crcsha1 ( + 'crc' TEXT NOT NULL, + 'sha1' TEXT NOT NULL, + PRIMARY KEY (crc, sha1) +)"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = @" +CREATE TABLE IF NOT EXISTS md5sha1 ( + 'md5' TEXT NOT NULL, + 'sha1' TEXT NOT NULL, + PRIMARY KEY (md5, sha1) +)"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); + + query = @" +CREATE TABLE IF NOT EXISTS dat ( 'hash' TEXT NOT NULL, - PRIMARY KEY (id, hash) + PRIMARY KEY (hash) )"; slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery();