diff --git a/RombaSharp/Partials/RombaSharp_Helpers.cs b/RombaSharp/Partials/RombaSharp_Helpers.cs index c430af3b..fbf602b5 100644 --- a/RombaSharp/Partials/RombaSharp_Helpers.cs +++ b/RombaSharp/Partials/RombaSharp_Helpers.cs @@ -13,6 +13,122 @@ namespace SabreTools { #region Helper methods + /// + /// Display the statistics in the database + /// + private static void DisplayDBStats() + { + SqliteConnection dbc = new SqliteConnection(_connectionString); + dbc.Open(); + + // Total number of CRCs + string query = "SELECT COUNT(*) FROM crc"; + SqliteCommand slc = new SqliteCommand(query, dbc); + _logger.User("Total CRCs: " + (long)slc.ExecuteScalar()); + + // Total number of MD5s + query = "SELECT COUNT(*) FROM md5"; + slc = new SqliteCommand(query, dbc); + _logger.User("Total MD5s: " + (long)slc.ExecuteScalar()); + + // Total number of SHA1s + query = "SELECT COUNT(*) FROM sha1"; + slc = new SqliteCommand(query, dbc); + _logger.User("Total SHA1s: " + (long)slc.ExecuteScalar()); + + // Total number of DATs + query = "SELECT COUNT(*) FROM dat"; + slc = new SqliteCommand(query, dbc); + _logger.User("Total DATs: " + (long)slc.ExecuteScalar()); + + slc.Dispose(); + dbc.Dispose(); + } + + /// + /// Display the current memory usage of the application + /// + private static void DisplayMemoryStats() + { + Process proc = Process.GetCurrentProcess(); + + _logger.User("Current Nonpaged Memory: " + Style.GetBytesReadable(proc.NonpagedSystemMemorySize64)); + _logger.User("Current Paged Memory: " + Style.GetBytesReadable(proc.PagedMemorySize64)); + _logger.User("Peak Paged Memory: " + Style.GetBytesReadable(proc.PeakPagedMemorySize64)); + _logger.User("Peak Virtual Memory: " + Style.GetBytesReadable(proc.PeakVirtualMemorySize64)); + _logger.User("Peak Working Memory: " + Style.GetBytesReadable(proc.PeakWorkingSet64)); + _logger.User("Private Memory: " + Style.GetBytesReadable(proc.PrivateMemorySize64)); + _logger.User("Virtual Memory: " + Style.GetBytesReadable(proc.VirtualMemorySize64)); + _logger.User("Working Memory: " + Style.GetBytesReadable(proc.WorkingSet64)); + _logger.User("Total Processor Time: " + proc.TotalProcessorTime); + _logger.User("User Processor Time: " + proc.UserProcessorTime); + } + + /// + /// Export the current database to CSV + /// + /// REDO + private static void ExportDatabase() + { + SqliteConnection dbc = new SqliteConnection(_connectionString); + dbc.Open(); + StreamWriter sw = new StreamWriter(File.Open("export.csv", FileMode.Create, FileAccess.Write)); + + sw.WriteLine("\"ID\",\"Size\",\"CRC\",\"MD5\",\"SHA-1\",\"In Depot\",\"DAT Hash\""); + + string query = "SELECT dats.id, size, crc, md5, sha1, indepot, hash FROM data JOIN dats ON data.id=dats.id"; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + + if (sldr.HasRows) + { + while (sldr.Read()) + { + string line = "\"" + sldr.GetInt32(0) + "\"," + + "\"" + sldr.GetInt64(1) + "\"," + + "\"" + sldr.GetString(2) + "\"," + + "\"" + sldr.GetString(3) + "\"," + + "\"" + sldr.GetString(4) + "\"," + + "\"" + sldr.GetInt32(5) + "\"," + + "\"" + sldr.GetString(6) + "\""; + sw.WriteLine(line); + } + } + + sldr.Dispose(); + slc.Dispose(); + sw.Dispose(); + dbc.Dispose(); + } + + /// + /// Gets all valid DATs that match in the DAT root + /// + /// List of input strings to check for, presumably file names + /// Dictionary of hash/full path for each of the valid DATs + private static Dictionary GetValidDats(List inputs) + { + // Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not + List datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList(); + List lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant()); + Dictionary foundDats = new Dictionary(); + foreach (string input in inputs) + { + if (lowerCaseDats.Contains(input.ToLowerInvariant())) + { + string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]); + string sha1 = FileTools.GetFileInfo(fullpath, _logger).SHA1; + foundDats.Add(sha1, fullpath); + } + else + { + _logger.Warning("The file '" + input + "' could not be found in the DAT root"); + } + } + + return foundDats; + } + /// /// Initialize the Romba application from XML config /// @@ -224,94 +340,6 @@ namespace SabreTools _port = port; } - /// - /// Display the statistics in the database - /// - private static void DisplayDBStats() - { - SqliteConnection dbc = new SqliteConnection(_connectionString); - dbc.Open(); - - // Total number of CRCs - string query = "SELECT COUNT(*) FROM crc"; - SqliteCommand slc = new SqliteCommand(query, dbc); - _logger.User("Total CRCs: " + (long)slc.ExecuteScalar()); - - // Total number of MD5s - query = "SELECT COUNT(*) FROM md5"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total MD5s: " + (long)slc.ExecuteScalar()); - - // Total number of SHA1s - query = "SELECT COUNT(*) FROM sha1"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total SHA1s: " + (long)slc.ExecuteScalar()); - - // Total number of DATs - query = "SELECT COUNT(*) FROM dat"; - slc = new SqliteCommand(query, dbc); - _logger.User("Total DATs: " + (long)slc.ExecuteScalar()); - - slc.Dispose(); - dbc.Dispose(); - } - - /// - /// Display the current memory usage of the application - /// - private static void DisplayMemoryStats() - { - Process proc = Process.GetCurrentProcess(); - - _logger.User("Current Nonpaged Memory: " + Style.GetBytesReadable(proc.NonpagedSystemMemorySize64)); - _logger.User("Current Paged Memory: " + Style.GetBytesReadable(proc.PagedMemorySize64)); - _logger.User("Peak Paged Memory: " + Style.GetBytesReadable(proc.PeakPagedMemorySize64)); - _logger.User("Peak Virtual Memory: " + Style.GetBytesReadable(proc.PeakVirtualMemorySize64)); - _logger.User("Peak Working Memory: " + Style.GetBytesReadable(proc.PeakWorkingSet64)); - _logger.User("Private Memory: " + Style.GetBytesReadable(proc.PrivateMemorySize64)); - _logger.User("Virtual Memory: " + Style.GetBytesReadable(proc.VirtualMemorySize64)); - _logger.User("Working Memory: " + Style.GetBytesReadable(proc.WorkingSet64)); - _logger.User("Total Processor Time: " + proc.TotalProcessorTime); - _logger.User("User Processor Time: " + proc.UserProcessorTime); - } - - /// - /// Export the current database to CSV - /// - /// REDO - private static void ExportDatabase() - { - SqliteConnection dbc = new SqliteConnection(_connectionString); - dbc.Open(); - StreamWriter sw = new StreamWriter(File.Open("export.csv", FileMode.Create, FileAccess.Write)); - - sw.WriteLine("\"ID\",\"Size\",\"CRC\",\"MD5\",\"SHA-1\",\"In Depot\",\"DAT Hash\""); - - string query = "SELECT dats.id, size, crc, md5, sha1, indepot, hash FROM data JOIN dats ON data.id=dats.id"; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - - if (sldr.HasRows) - { - while (sldr.Read()) - { - string line = "\"" + sldr.GetInt32(0) + "\"," - + "\"" + sldr.GetInt64(1) + "\"," - + "\"" + sldr.GetString(2) + "\"," - + "\"" + sldr.GetString(3) + "\"," - + "\"" + sldr.GetString(4) + "\"," - + "\"" + sldr.GetInt32(5) + "\"," - + "\"" + sldr.GetString(6) + "\""; - sw.WriteLine(line); - } - } - - sldr.Dispose(); - slc.Dispose(); - sw.Dispose(); - dbc.Dispose(); - } - /// /// Moves DAT index entries for orphaned DATs to backup folder /// @@ -509,31 +537,69 @@ namespace SabreTools } /// - /// Gets all valid DATs that match in the DAT root + /// Rescan a particular depot path into the database /// - /// List of input strings to check for, presumably file names - /// Dictionary of hash/full path for each of the valid DATs - private static Dictionary GetValidDats(List inputs) + /// Path to the depot to be rescanned + private static void Rescan(string depotname) { - // Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not - List datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList(); - List lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant()); - Dictionary foundDats = new Dictionary(); - foreach (string input in inputs) + // Check that it's a valid depot first + if (!_depots.ContainsKey(depotname)) { - if (lowerCaseDats.Contains(input.ToLowerInvariant())) + _logger.User("'" + depotname + "' is not a recognized depot. Please add it to your configuration file and try again"); + return; + } + + // Then check that the depot is online + if (!Directory.Exists(depotname)) + { + _logger.User("'" + depotname + "' does not appear to be online. Please check its status and try again"); + return; + } + + // Open the database connection + SqliteConnection dbc = new SqliteConnection(_connectionString); + dbc.Open(); + + // If we have it, then check for all hashes that are in that depot + List hashes = new List(); + string query = "SELECT sha1 FROM sha1 WHERE depot=\"" + depotname + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + if (sldr.HasRows) + { + while (sldr.Read()) { - string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]); - string sha1 = FileTools.GetFileInfo(fullpath, _logger).SHA1; - foundDats.Add(sha1, fullpath); - } - else - { - _logger.Warning("The file '" + input + "' could not be found in the DAT root"); + hashes.Add(sldr.GetString(0)); } } - return foundDats; + // Now rescan the depot itself + DatFile depot = new DatFile(); + depot.PopulateDatFromDir(depotname, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger); + depot.BucketBySHA1(false, _logger, false); + + // Once we have both, check for any new files + List dupehashes = new List(); + List keys = depot.Files.Keys.ToList(); + foreach (string key in keys) + { + List roms = depot.Files[key]; + foreach (Rom rom in roms) + { + if (hashes.Contains(rom.SHA1)) + { + dupehashes.Add(rom.SHA1); + hashes.Remove(rom.SHA1); + } + else if (!dupehashes.Contains(rom.SHA1)) + { + + } + } + } + + // Dispose of the database connection + dbc.Dispose(); } #endregion diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index 360deb4b..4381d73e 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -1,5 +1,6 @@ using Mono.Data.Sqlite; using SabreTools.Helper; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -38,61 +39,97 @@ namespace SabreTools DatFile need = new DatFile(); need.Files = new SortedDictionary>(); + // Open the database connection SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // Now that we have the Dats, add the files to the database + string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; + string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; + string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES"; + string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; + string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; + foreach (string key in df.Files.Keys) { List datItems = df.Files[key]; foreach (Rom rom in datItems) { - string query = "SELECT id FROM data WHERE size=" + rom.Size - + " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")" - + " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")" - + " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")" - + " AND indepot=0"; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - - // If a row is returned, add the file and change the existence - if (sldr.HasRows) + // If we care about if the file exists, check the databse first + if (onlyNeeded) { - sldr.Read(); - long id = sldr.GetInt64(0); - - string squery = "UPDATE data SET indepot=1 WHERE id=" + id; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - sslc.Dispose(); - - // Add the rom to the files that need to be rebuilt - if (need.Files.ContainsKey(key)) + string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1" + + " WHERE crcsha1.crc=\"" + rom.CRC + "\"" + + " OR md5sha1.md5=\"" + rom.MD5 + "\"" + + " OR md5sha1.sha1=\"" + rom.SHA1 + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + + if (sldr.HasRows) { - need.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - need.Files.Add(key, temp); + // Add to the queries + if (!String.IsNullOrEmpty(rom.CRC)) + { + crcquery += " ('" + rom.CRC + "'),"; + } + if (!String.IsNullOrEmpty(rom.MD5)) + { + md5query += " ('" + rom.MD5 + "'),"; + } + if (!String.IsNullOrEmpty(rom.SHA1)) + { + sha1query += " ('" + rom.SHA1 + "'),"; + + if (!String.IsNullOrEmpty(rom.CRC)) + { + crcsha1query += " ('" + rom.CRC + "', '" + rom.SHA1 + "'),"; + } + if (!String.IsNullOrEmpty(rom.MD5)) + { + md5sha1query += " ('" + rom.MD5 + "', '" + rom.SHA1 + "'),"; + } + } + + // Add to the Dat + if (need.Files.ContainsKey(key)) + { + need.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + need.Files.Add(key, temp); + } } } - - // If it doesn't exist, and we're not adding only needed files - else if (!onlyNeeded) + // Otherwise, just add the file to the list + else { - string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES (" - + rom.Size + "," - + "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," - + "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," - + "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," - + "1)"; - SqliteCommand sslc = new SqliteCommand(squery, dbc); - sslc.ExecuteNonQuery(); - sslc.Dispose(); + // Add to the queries + if (!String.IsNullOrEmpty(rom.CRC)) + { + crcquery += " ('" + rom.CRC + "'),"; + } + if (!String.IsNullOrEmpty(rom.MD5)) + { + md5query += " ('" + rom.MD5 + "'),"; + } + if (!String.IsNullOrEmpty(rom.SHA1)) + { + sha1query += " ('" + rom.SHA1 + "'),"; + + if (!String.IsNullOrEmpty(rom.CRC)) + { + crcsha1query += " ('" + rom.CRC + "', '" + rom.SHA1 + "'),"; + } + if (!String.IsNullOrEmpty(rom.MD5)) + { + md5sha1query += " ('" + rom.MD5 + "', '" + rom.SHA1 + "'),"; + } + } - // Add the rom to the files that need to be rebuilt + // Add to the Dat if (need.Files.ContainsKey(key)) { need.Files[key].Add(rom); @@ -107,6 +144,38 @@ namespace SabreTools } } + // Now run the queries, if they're populated + if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") + { + SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + slc.Dispose(); + } + if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") + { + SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + slc.Dispose(); + } + if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES") + { + SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + slc.Dispose(); + } + if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") + { + SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + slc.Dispose(); + } + if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") + { + SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); + slc.ExecuteNonQuery(); + slc.Dispose(); + } + // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(0, 0, 0, 0); SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, false, true, true, asl, false, _logger); @@ -129,10 +198,6 @@ namespace SabreTools Directory.CreateDirectory("out"); } - // Open the database - SqliteConnection dbc = new SqliteConnection(_connectionString); - dbc.Open(); - // Now that we have the dictionary, we can loop through and output to a new folder for each foreach (string key in foundDats.Keys) { @@ -148,60 +213,13 @@ namespace SabreTools Directory.CreateDirectory(outputFolder); } - // Then get all hashes associated with this DAT - string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\""; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - if (sldr.HasRows) - { - while (sldr.Read()) - { - string sha1 = sldr.GetString(0); - string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz"); + // Get all online depots + List onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList(); - // Find the first depot that contains the folder - foreach (string depot in _depots.Keys) - { - // If the depot is online, check it - if (_depots[depot].Item2) - { - if (File.Exists(Path.Combine(depot, filename))) - { - if (copy) - { - if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename)))) - { - Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename))); - } - - try - { - File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true); - } - catch { } - } - else - { - ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger); - } - continue; - } - } - } - } - } - - // Now that we have extracted everything, we rebuild to the output folder - if (!copy) - { - List temp = new List(); - temp.Add(_tmpdir); - SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, false, true, false, false, asl, false, _logger); - ss.StartProcessing(); - } + // Now scan all of those depots and rebuild + SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, false, copy, copy, asl, false, _logger); + ss.StartProcessing(); } - - dbc.Dispose(); } /// @@ -305,7 +323,7 @@ namespace SabreTools // Now, search for each of them and return true or false for each foreach (string input in crc) { - string query = "SELECT * FROM data WHERE crc=\"" + input + "\""; + string query = "SELECT * FROM crc WHERE crc=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -322,7 +340,7 @@ namespace SabreTools } foreach (string input in md5) { - string query = "SELECT * FROM data WHERE md5=\"" + input + "\""; + string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -339,7 +357,7 @@ namespace SabreTools } foreach (string input in sha1) { - string query = "SELECT * FROM data WHERE sha1=\"" + input + "\""; + string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) diff --git a/SabreTools.Helper/Objects/Dat/DatFile.cs b/SabreTools.Helper/Objects/Dat/DatFile.cs index 34b88947..0f17e4d1 100644 --- a/SabreTools.Helper/Objects/Dat/DatFile.cs +++ b/SabreTools.Helper/Objects/Dat/DatFile.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.Serialization.Formatters.Binary; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; diff --git a/SabreTools.Helper/Tools/DatabaseTools.cs b/SabreTools.Helper/Tools/DatabaseTools.cs index 3f9757f0..d0006ffd 100644 --- a/SabreTools.Helper/Tools/DatabaseTools.cs +++ b/SabreTools.Helper/Tools/DatabaseTools.cs @@ -90,6 +90,7 @@ CREATE TABLE IF NOT EXISTS md5 ( query = @" CREATE TABLE IF NOT EXISTS sha1 ( 'sha1' TEXT NOT NULL, + 'depot' TEXT, PRIMARY KEY (sha1) )"; slc = new SqliteCommand(query, dbc);