claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[RombaSharp] Recreate lost KV stuff

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-17 11:04:07 -07:00
parent d6cd471999
commit 6de1c64371
4 changed files with 290 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
RombaSharp/Partials/RombaSharp_Helpers.cs
View File

@@ -13,6 +13,122 @@ namespace SabreTools
{
#region Helper methods
/// <summary>
/// Display the statistics in the database
/// </summary>
private static void DisplayDBStats()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Total number of CRCs
string query = "SELECT COUNT(*) FROM crc";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Total CRCs: " + (long)slc.ExecuteScalar());
// Total number of MD5s
query = "SELECT COUNT(*) FROM md5";
slc = new SqliteCommand(query, dbc);
_logger.User("Total MD5s: " + (long)slc.ExecuteScalar());
// Total number of SHA1s
query = "SELECT COUNT(*) FROM sha1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total SHA1s: " + (long)slc.ExecuteScalar());
// Total number of DATs
query = "SELECT COUNT(*) FROM dat";
slc = new SqliteCommand(query, dbc);
_logger.User("Total DATs: " + (long)slc.ExecuteScalar());
slc.Dispose();
dbc.Dispose();
}
/// <summary>
/// Display the current memory usage of the application
/// </summary>
private static void DisplayMemoryStats()
{
Process proc = Process.GetCurrentProcess();
_logger.User("Current Nonpaged Memory: " + Style.GetBytesReadable(proc.NonpagedSystemMemorySize64));
_logger.User("Current Paged Memory: " + Style.GetBytesReadable(proc.PagedMemorySize64));
_logger.User("Peak Paged Memory: " + Style.GetBytesReadable(proc.PeakPagedMemorySize64));
_logger.User("Peak Virtual Memory: " + Style.GetBytesReadable(proc.PeakVirtualMemorySize64));
_logger.User("Peak Working Memory: " + Style.GetBytesReadable(proc.PeakWorkingSet64));
_logger.User("Private Memory: " + Style.GetBytesReadable(proc.PrivateMemorySize64));
_logger.User("Virtual Memory: " + Style.GetBytesReadable(proc.VirtualMemorySize64));
_logger.User("Working Memory: " + Style.GetBytesReadable(proc.WorkingSet64));
_logger.User("Total Processor Time: " + proc.TotalProcessorTime);
_logger.User("User Processor Time: " + proc.UserProcessorTime);
}
/// <summary>
/// Export the current database to CSV
/// </summary>
/// <remarks>REDO</remarks>
private static void ExportDatabase()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
StreamWriter sw = new StreamWriter(File.Open("export.csv", FileMode.Create, FileAccess.Write));
sw.WriteLine("\"ID\",\"Size\",\"CRC\",\"MD5\",\"SHA-1\",\"In Depot\",\"DAT Hash\"");
string query = "SELECT dats.id, size, crc, md5, sha1, indepot, hash FROM data JOIN dats ON data.id=dats.id";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string line = "\"" + sldr.GetInt32(0) + "\","
+ "\"" + sldr.GetInt64(1) + "\","
+ "\"" + sldr.GetString(2) + "\","
+ "\"" + sldr.GetString(3) + "\","
+ "\"" + sldr.GetString(4) + "\","
+ "\"" + sldr.GetInt32(5) + "\","
+ "\"" + sldr.GetString(6) + "\"";
sw.WriteLine(line);
}
}
sldr.Dispose();
slc.Dispose();
sw.Dispose();
dbc.Dispose();
}
/// <summary>
/// Gets all valid DATs that match in the DAT root
/// </summary>
/// <param name="inputs">List of input strings to check for, presumably file names</param>
/// <returns>Dictionary of hash/full path for each of the valid DATs</returns>
private static Dictionary<string, string> GetValidDats(List<string> inputs)
{
// Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not
List<string> datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList();
List<string> lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant());
Dictionary<string, string> foundDats = new Dictionary<string, string>();
foreach (string input in inputs)
{
if (lowerCaseDats.Contains(input.ToLowerInvariant()))
{
string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]);
string sha1 = FileTools.GetFileInfo(fullpath, _logger).SHA1;
foundDats.Add(sha1, fullpath);
}
else
{
_logger.Warning("The file '" + input + "' could not be found in the DAT root");
}
}
return foundDats;
}
/// <summary>
/// Initialize the Romba application from XML config
/// </summary>
@@ -224,94 +340,6 @@ namespace SabreTools
_port = port;
}
/// <summary>
/// Display the statistics in the database
/// </summary>
private static void DisplayDBStats()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Total number of CRCs
string query = "SELECT COUNT(*) FROM crc";
SqliteCommand slc = new SqliteCommand(query, dbc);
_logger.User("Total CRCs: " + (long)slc.ExecuteScalar());
// Total number of MD5s
query = "SELECT COUNT(*) FROM md5";
slc = new SqliteCommand(query, dbc);
_logger.User("Total MD5s: " + (long)slc.ExecuteScalar());
// Total number of SHA1s
query = "SELECT COUNT(*) FROM sha1";
slc = new SqliteCommand(query, dbc);
_logger.User("Total SHA1s: " + (long)slc.ExecuteScalar());
// Total number of DATs
query = "SELECT COUNT(*) FROM dat";
slc = new SqliteCommand(query, dbc);
_logger.User("Total DATs: " + (long)slc.ExecuteScalar());
slc.Dispose();
dbc.Dispose();
}
/// <summary>
/// Display the current memory usage of the application
/// </summary>
private static void DisplayMemoryStats()
{
Process proc = Process.GetCurrentProcess();
_logger.User("Current Nonpaged Memory: " + Style.GetBytesReadable(proc.NonpagedSystemMemorySize64));
_logger.User("Current Paged Memory: " + Style.GetBytesReadable(proc.PagedMemorySize64));
_logger.User("Peak Paged Memory: " + Style.GetBytesReadable(proc.PeakPagedMemorySize64));
_logger.User("Peak Virtual Memory: " + Style.GetBytesReadable(proc.PeakVirtualMemorySize64));
_logger.User("Peak Working Memory: " + Style.GetBytesReadable(proc.PeakWorkingSet64));
_logger.User("Private Memory: " + Style.GetBytesReadable(proc.PrivateMemorySize64));
_logger.User("Virtual Memory: " + Style.GetBytesReadable(proc.VirtualMemorySize64));
_logger.User("Working Memory: " + Style.GetBytesReadable(proc.WorkingSet64));
_logger.User("Total Processor Time: " + proc.TotalProcessorTime);
_logger.User("User Processor Time: " + proc.UserProcessorTime);
}
/// <summary>
/// Export the current database to CSV
/// </summary>
/// <remarks>REDO</remarks>
private static void ExportDatabase()
{
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
StreamWriter sw = new StreamWriter(File.Open("export.csv", FileMode.Create, FileAccess.Write));
sw.WriteLine("\"ID\",\"Size\",\"CRC\",\"MD5\",\"SHA-1\",\"In Depot\",\"DAT Hash\"");
string query = "SELECT dats.id, size, crc, md5, sha1, indepot, hash FROM data JOIN dats ON data.id=dats.id";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string line = "\"" + sldr.GetInt32(0) + "\","
+ "\"" + sldr.GetInt64(1) + "\","
+ "\"" + sldr.GetString(2) + "\","
+ "\"" + sldr.GetString(3) + "\","
+ "\"" + sldr.GetString(4) + "\","
+ "\"" + sldr.GetInt32(5) + "\","
+ "\"" + sldr.GetString(6) + "\"";
sw.WriteLine(line);
}
}
sldr.Dispose();
slc.Dispose();
sw.Dispose();
dbc.Dispose();
}
/// <summary>
/// Moves DAT index entries for orphaned DATs to backup folder
/// </summary>
@@ -509,31 +537,69 @@ namespace SabreTools
}
/// <summary>
/// Gets all valid DATs that match in the DAT root
/// Rescan a particular depot path into the database
/// </summary>
/// <param name="inputs">List of input strings to check for, presumably file names</param>
/// <returns>Dictionary of hash/full path for each of the valid DATs</returns>
private static Dictionary<string, string> GetValidDats(List<string> inputs)
/// <param name="depotname">Path to the depot to be rescanned</param>
private static void Rescan(string depotname)
{
// Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not
List<string> datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList();
List<string> lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant());
Dictionary<string, string> foundDats = new Dictionary<string, string>();
foreach (string input in inputs)
// Check that it's a valid depot first
if (!_depots.ContainsKey(depotname))
{
if (lowerCaseDats.Contains(input.ToLowerInvariant()))
{
string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]);
string sha1 = FileTools.GetFileInfo(fullpath, _logger).SHA1;
foundDats.Add(sha1, fullpath);
_logger.User("'" + depotname + "' is not a recognized depot. Please add it to your configuration file and try again");
return;
}
else
// Then check that the depot is online
if (!Directory.Exists(depotname))
{
_logger.Warning("The file '" + input + "' could not be found in the DAT root");
_logger.User("'" + depotname + "' does not appear to be online. Please check its status and try again");
return;
}
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// If we have it, then check for all hashes that are in that depot
List<string> hashes = new List<string>();
string query = "SELECT sha1 FROM sha1 WHERE depot=\"" + depotname + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
hashes.Add(sldr.GetString(0));
}
}
return foundDats;
// Now rescan the depot itself
DatFile depot = new DatFile();
depot.PopulateDatFromDir(depotname, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger);
depot.BucketBySHA1(false, _logger, false);
// Once we have both, check for any new files
List<string> dupehashes = new List<string>();
List<string> keys = depot.Files.Keys.ToList();
foreach (string key in keys)
{
List<DatItem> roms = depot.Files[key];
foreach (Rom rom in roms)
{
if (hashes.Contains(rom.SHA1))
{
dupehashes.Add(rom.SHA1);
hashes.Remove(rom.SHA1);
}
else if (!dupehashes.Contains(rom.SHA1))
{
}
}
}
// Dispose of the database connection
dbc.Dispose();
}
#endregion

204
RombaSharp/Partials/RombaSharp_Inits.cs
View File

@@ -1,5 +1,6 @@
using Mono.Data.Sqlite;
using SabreTools.Helper;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -38,61 +39,58 @@ namespace SabreTools
DatFile need = new DatFile();
need.Files = new SortedDictionary<string, List<DatItem>>();
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the Dats, add the files to the database
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
foreach (string key in df.Files.Keys)
{
List<DatItem> datItems = df.Files[key];
foreach (Rom rom in datItems)
{
string query = "SELECT id FROM data WHERE size=" + rom.Size
+ " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")"
+ " AND indepot=0";
// If we care about if the file exists, check the databse first
if (onlyNeeded)
{
string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
+ " WHERE crcsha1.crc=\"" + rom.CRC + "\""
+ " OR md5sha1.md5=\"" + rom.MD5 + "\""
+ " OR md5sha1.sha1=\"" + rom.SHA1 + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
// If a row is returned, add the file and change the existence
if (sldr.HasRows)
{
sldr.Read();
long id = sldr.GetInt64(0);
string squery = "UPDATE data SET indepot=1 WHERE id=" + id;
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
// Add the rom to the files that need to be rebuilt
if (need.Files.ContainsKey(key))
// Add to the queries
if (!String.IsNullOrEmpty(rom.CRC))
{
need.Files[key].Add(rom);
crcquery += " ('" + rom.CRC + "'),";
}
else
if (!String.IsNullOrEmpty(rom.MD5))
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
need.Files.Add(key, temp);
md5query += " ('" + rom.MD5 + "'),";
}
if (!String.IsNullOrEmpty(rom.SHA1))
{
sha1query += " ('" + rom.SHA1 + "'),";
if (!String.IsNullOrEmpty(rom.CRC))
{
crcsha1query += " ('" + rom.CRC + "', '" + rom.SHA1 + "'),";
}
if (!String.IsNullOrEmpty(rom.MD5))
{
md5sha1query += " ('" + rom.MD5 + "', '" + rom.SHA1 + "'),";
}
}
// If it doesn't exist, and we're not adding only needed files
else if (!onlyNeeded)
{
string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+ rom.Size + ","
+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+ "1)";
SqliteCommand sslc = new SqliteCommand(squery, dbc);
sslc.ExecuteNonQuery();
sslc.Dispose();
// Add the rom to the files that need to be rebuilt
// Add to the Dat
if (need.Files.ContainsKey(key))
{
need.Files[key].Add(rom);
@@ -105,6 +103,77 @@ namespace SabreTools
}
}
}
// Otherwise, just add the file to the list
else
{
// Add to the queries
if (!String.IsNullOrEmpty(rom.CRC))
{
crcquery += " ('" + rom.CRC + "'),";
}
if (!String.IsNullOrEmpty(rom.MD5))
{
md5query += " ('" + rom.MD5 + "'),";
}
if (!String.IsNullOrEmpty(rom.SHA1))
{
sha1query += " ('" + rom.SHA1 + "'),";
if (!String.IsNullOrEmpty(rom.CRC))
{
crcsha1query += " ('" + rom.CRC + "', '" + rom.SHA1 + "'),";
}
if (!String.IsNullOrEmpty(rom.MD5))
{
md5sha1query += " ('" + rom.MD5 + "', '" + rom.SHA1 + "'),";
}
}
// Add to the Dat
if (need.Files.ContainsKey(key))
{
need.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
need.Files.Add(key, temp);
}
}
}
}
// Now run the queries, if they're populated
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
// Create the sorting object to use and rebuild the needed files
@@ -129,10 +198,6 @@ namespace SabreTools
Directory.CreateDirectory("out");
}
// Open the database
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the dictionary, we can loop through and output to a new folder for each
foreach (string key in foundDats.Keys)
{
@@ -148,62 +213,15 @@ namespace SabreTools
Directory.CreateDirectory(outputFolder);
}
// Then get all hashes associated with this DAT
string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string sha1 = sldr.GetString(0);
string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz");
// Get all online depots
List<string> onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList();
// Find the first depot that contains the folder
foreach (string depot in _depots.Keys)
{
// If the depot is online, check it
if (_depots[depot].Item2)
{
if (File.Exists(Path.Combine(depot, filename)))
{
if (copy)
{
if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename))))
{
Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename)));
}
try
{
File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true);
}
catch { }
}
else
{
ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger);
}
continue;
}
}
}
}
}
// Now that we have extracted everything, we rebuild to the output folder
if (!copy)
{
List<string> temp = new List<string>();
temp.Add(_tmpdir);
SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, false, true, false, false, asl, false, _logger);
// Now scan all of those depots and rebuild
SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, false, copy, copy, asl, false, _logger);
ss.StartProcessing();
}
}
dbc.Dispose();
}
/// <summary>
/// Wrap finding all files that are in both the database and a new Dat
/// </summary>
@@ -305,7 +323,7 @@ namespace SabreTools
// Now, search for each of them and return true or false for each
foreach (string input in crc)
{
string query = "SELECT * FROM data WHERE crc=\"" + input + "\"";
string query = "SELECT * FROM crc WHERE crc=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -322,7 +340,7 @@ namespace SabreTools
}
foreach (string input in md5)
{
string query = "SELECT * FROM data WHERE md5=\"" + input + "\"";
string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
@@ -339,7 +357,7 @@ namespace SabreTools
}
foreach (string input in sha1)
{
string query = "SELECT * FROM data WHERE sha1=\"" + input + "\"";
string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)

1
SabreTools.Helper/Objects/Dat/DatFile.cs
View File

@@ -3,7 +3,6 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

1
SabreTools.Helper/Tools/DatabaseTools.cs
View File

@@ -90,6 +90,7 @@ CREATE TABLE IF NOT EXISTS md5 (
query = @"
CREATE TABLE IF NOT EXISTS sha1 (
'sha1' TEXT NOT NULL,
'depot' TEXT,
PRIMARY KEY (sha1)
)";
slc = new SqliteCommand(query, dbc);