diff --git a/RombaSharp/Partials/RombaSharp_Helpers.cs b/RombaSharp/Partials/RombaSharp_Helpers.cs index 411d322b..7489006c 100644 --- a/RombaSharp/Partials/RombaSharp_Helpers.cs +++ b/RombaSharp/Partials/RombaSharp_Helpers.cs @@ -313,7 +313,7 @@ namespace SabreTools List databaseDats = new List(); // Populate the List from the database - string query = "SELECT UNIQUE value FROM data WHERE key=\"dat\""; + string query = "SELECT UNIQUE hash FROM dats"; SqliteConnection dbc = new SqliteConnection(_connectionString); SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); @@ -366,29 +366,44 @@ namespace SabreTools { foreach (Rom rom in tempdat.Files[romkey]) { - query = "SELECT id FROM data WHERE key=\"size\" AND value=\"" + rom.Size + "\" AND (" - + "(key=\"crc\" AND (value=\"" + rom.CRC + "\" OR value=\"null\"))" - + "AND (key=\"md5\" AND value=\"" + rom.MD5 + "\" OR value=\"null\"))" - + "AND (key=\"sha1\" AND value=\"" + rom.SHA1 + "\" OR value=\"null\")))"; + query = "SELECT id FROM data WHERE size=" + rom.Size + " AND (" + + "(crc=\"" + rom.CRC + "\" OR value=\"null\")" + + " AND (md5=\"" + rom.MD5 + "\" OR value=\"null\")" + + " AND (sha1=\"" + rom.SHA1 + "\" OR value=\"null\"))"; slc = new SqliteCommand(query, dbc); sldr = slc.ExecuteReader(); - // If the hash exists in the database, add the dat hash for that id + // If the hash exists in the database, add the dat hash for that id if needed if (sldr.HasRows) { sldr.Read(); string id = sldr.GetString(0); - string squery = "INSERT INTO data (id, key, value) VALUES (\"" + id + "\", \"dat\", \"" + key + "\")"; + string squery = "SELECT * FROM dats WHERE id=" + id; SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); + SqliteDataReader ssldr = sslc.ExecuteReader(); + + // If the hash doesn't already exist, add it + if (!ssldr.HasRows) + { + squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")"; + sslc = new SqliteCommand(squery, dbc); + sslc.ExecuteNonQuery(); + } + + ssldr.Dispose(); sslc.Dispose(); } // If it doesn't exist, add the hash and the dat hash for a new id else { - string squery = "INSERT INTO data (key, value) VALUES (\"size\", \"" + rom.Size + "\")"; + string squery = "INSERT INTO data (size, crc, md5, sha1, exists) VALUES" + + " size=" + rom.Size + "," + + " crc=\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," + + " md5=\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," + + " sha1=\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," + + " exists=0)"; SqliteCommand sslc = new SqliteCommand(squery, dbc); sslc.ExecuteNonQuery(); @@ -398,13 +413,10 @@ namespace SabreTools sslc = new SqliteCommand(squery, dbc); id = (long)sslc.ExecuteScalar(); - squery = "INSERT INTO data (id, key, value) VALUES (\"" + id + "\", \"crc\", \"" + rom.CRC + "\")," - + " (\"" + id + "\", \"md5\", \"" + rom.MD5 + "\")," - + " (\"" + id + "\", \"sha1\", \"" + rom.SHA1 + "\")," - + " (\"" + id + "\", \"dat\", \"" + key + "\")," - + " (\"" + id + "\", \"exists\", \"false\")"; + squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")"; sslc = new SqliteCommand(squery, dbc); sslc.ExecuteNonQuery(); + sslc.Dispose(); } } } @@ -414,12 +426,14 @@ namespace SabreTools // TODO: Remove orphaned files as well foreach (string dathash in databaseDats) { - query = "DELETE FROM data WHERE key=\"dat\" AND value=\"" + dathash + "\""; + query = "DELETE FROM dats WHERE hash=\"" + dathash + "\""; slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); slc.Dispose(); } } + + dbc.Dispose(); } /// diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index 863ed2e9..e813145e 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -1,8 +1,8 @@ using Mono.Data.Sqlite; using SabreTools.Helper; -using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace SabreTools { @@ -13,16 +13,102 @@ namespace SabreTools /// /// Wrap adding files to the depots /// - /// - /// + /// List of input folders to use + /// True if only files in the database and don't exist are added, false otherwise private static void InitArchive(List inputs, bool onlyNeeded) { - _logger.User("This feature is not yet implemented: archive"); + // First we want to get just all directories from the inputs + List onlyDirs = new List(); + foreach (string input in inputs) + { + if (Directory.Exists(input)) + { + onlyDirs.Add(Path.GetFullPath(input)); + } + } - // This should use the same thing as something like in SimpleSort, as in scan the archive and scan the files inside equally - // Either during or after, we then check against the database to see if there's any matches. If there's no match and - // we say onlyNeeded, then don't add it. If there's no match and not onlyNeeded, add it with no DAT hash. If there's a match, - // and it doesn't say exists, add it and change the flag. If there's a match and says exists, skip it. + // Then process all of the input directories into an internal DAT + DatFile df = new DatFile(); + foreach (string dir in onlyDirs) + { + df.PopulateDatFromDir(dir, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger); + } + + // Create an empty Dat for files that need to be rebuilt + DatFile need = new DatFile(); + need.Files = new SortedDictionary>(); + + // Now that we have the Dats, add the files to the database + SqliteConnection dbc = new SqliteConnection(_connectionString); + foreach (string key in df.Files.Keys) + { + List datItems = df.Files[key]; + foreach (Rom rom in datItems) + { + string query = "SELECT id FROM data WHERE size=" + rom.Size + " AND (" + + "(crc=\"" + rom.CRC + "\" OR value=\"null\")" + + " AND (md5=\"" + rom.MD5 + "\" OR value=\"null\")" + + " AND (sha1=\"" + rom.SHA1 + "\" OR value=\"null\")" + + " AND exists=0"; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + + // If a row is returned, add the file and change the existence + if (sldr.HasRows) + { + sldr.Read(); + string id = sldr.GetString(0); + + string squery = "UPDATE data SET exists=1 WHERE id=" + id; + SqliteCommand sslc = new SqliteCommand(squery, dbc); + sslc.ExecuteNonQuery(); + sslc.Dispose(); + + // Add the rom to the files that need to be rebuilt + if (need.Files.ContainsKey(key)) + { + need.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + need.Files.Add(key, temp); + } + } + + // If it doesn't exist, and we're not adding only needed files + else if (!onlyNeeded) + { + string squery = "INSERT INTO data (size, crc, md5, sha1, exists) VALUES" + + " size=" + rom.Size + "," + + " crc=\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," + + " md5=\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," + + " sha1=\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," + + " exists=1)"; + SqliteCommand sslc = new SqliteCommand(squery, dbc); + sslc.ExecuteNonQuery(); + sslc.Dispose(); + + // Add the rom to the files that need to be rebuilt + if (need.Files.ContainsKey(key)) + { + need.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + need.Files.Add(key, temp); + } + } + } + } + + // Create the sorting object to use and rebuild the needed files + ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(0, 0, 0, 0); + SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, _logger); + ss.StartProcessing(); } /// @@ -99,40 +185,81 @@ namespace SabreTools /// /// Wrap looking up if hashes exist in the database /// - /// + /// List of input strings representing hashes to check for private static void InitLookup(List inputs) { // First, try to figure out what type of hash each is by length and clean it - List cleanedInputs = new List(); + List crc = new List(); + List md5 = new List(); + List sha1 = new List(); foreach (string input in inputs) { string temp = ""; if (input.Length == Constants.CRCLength) { temp = Style.CleanHashData(input, Constants.CRCLength); + if (temp != "") + { + crc.Add(temp); + } } else if (input.Length == Constants.MD5Length) { temp = Style.CleanHashData(input, Constants.MD5Length); + if (temp != "") + { + md5.Add(temp); + } } else if (input.Length == Constants.SHA1Length) { temp = Style.CleanHashData(input, Constants.SHA1Length); - } - - // If the hash isn't empty, add it - if (temp != "") - { - cleanedInputs.Add(temp); + if (temp != "") + { + sha1.Add(temp); + } } } // Now, search for each of them and return true or false for each SqliteConnection dbc = new SqliteConnection(_connectionString); - - foreach (string input in cleanedInputs) + foreach (string input in crc) { - string query = "SELECT * FROM data WHERE value=\"" + input + "\""; + string query = "SELECT * FROM data WHERE crc=\"" + input + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + if (sldr.HasRows) + { + _logger.User("For hash '" + input + "' there were " + sldr.RecordsAffected + " matches in the database"); + } + else + { + _logger.User("Hash '" + input + "' had no matches in the database"); + } + + sldr.Dispose(); + slc.Dispose(); + } + foreach (string input in md5) + { + string query = "SELECT * FROM data WHERE md5=\"" + input + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + if (sldr.HasRows) + { + _logger.User("For hash '" + input + "' there were " + sldr.RecordsAffected + " matches in the database"); + } + else + { + _logger.User("Hash '" + input + "' had no matches in the database"); + } + + sldr.Dispose(); + slc.Dispose(); + } + foreach (string input in sha1) + { + string query = "SELECT * FROM data WHERE sha1=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) diff --git a/SabreTools.Helper/Tools/DatabaseTools.cs b/SabreTools.Helper/Tools/DatabaseTools.cs index cfa8f670..1455080f 100644 --- a/SabreTools.Helper/Tools/DatabaseTools.cs +++ b/SabreTools.Helper/Tools/DatabaseTools.cs @@ -73,13 +73,25 @@ namespace SabreTools.Helper { string query = @" CREATE TABLE IF NOT EXISTS data ( - 'id' INTEGER NOT NULL - 'key' TEXT NOT NULL - 'value' TEXT NOT NULL - PRIMARY KEY (id, key, value) + 'id' INTEGER NOT NULL + 'size' INTEGER NOT NULL + 'crc' TEXT NOT NULL + 'md5' TEXT NOT NULL + 'sha1' TEXT NOT NULL + 'exists' INTEGER NOT NULL + PRIMARY KEY (id) )"; SqliteCommand slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); + + query = @" +CREATE TABLE IF NOT EXISTS dats ( + 'id' INTEGER NOT NULL + 'hash' TEXT NOT NULL + PRIMARY KEY (id, hash) +)"; + slc = new SqliteCommand(query, dbc); + slc.ExecuteNonQuery(); slc.Dispose(); } else if (type == "headerer")