[RombaSharp] More RombaSharp updates; add Archive

RombaSharp/Partials/RombaSharp_Helpers.cs

@@ -313,7 +313,7 @@ namespace SabreTools
 			List<string> databaseDats = new List<string>();
 
 			// Populate the List from the database
-			string query = "SELECT UNIQUE value FROM data WHERE key=\"dat\"";
+			string query = "SELECT UNIQUE hash FROM dats";
 			SqliteConnection dbc = new SqliteConnection(_connectionString);
 			SqliteCommand slc = new SqliteCommand(query, dbc);
 			SqliteDataReader sldr = slc.ExecuteReader();
@@ -366,29 +366,44 @@ namespace SabreTools
 					{
 						foreach (Rom rom in tempdat.Files[romkey])
 						{
-							query = "SELECT id FROM data WHERE key=\"size\" AND value=\"" + rom.Size + "\" AND ("
+							query = "SELECT id FROM data WHERE size=" + rom.Size + " AND ("
-								+ "(key=\"crc\" AND (value=\"" + rom.CRC + "\" OR value=\"null\"))"
+								+ "(crc=\"" + rom.CRC + "\" OR value=\"null\")"
-								+ "AND (key=\"md5\" AND value=\"" + rom.MD5 + "\" OR value=\"null\"))"
+								+ " AND (md5=\"" + rom.MD5 + "\" OR value=\"null\")"
-								+ "AND (key=\"sha1\" AND value=\"" + rom.SHA1 + "\" OR value=\"null\")))";
+								+ " AND (sha1=\"" + rom.SHA1 + "\" OR value=\"null\"))";
 							slc = new SqliteCommand(query, dbc);
 							sldr = slc.ExecuteReader();
 								
-							// If the hash exists in the database, add the dat hash for that id
+							// If the hash exists in the database, add the dat hash for that id if needed
 							if (sldr.HasRows)
 							{
 								sldr.Read();
 								string id = sldr.GetString(0);
 
-								string squery = "INSERT INTO data (id, key, value) VALUES (\"" + id + "\", \"dat\", \"" + key + "\")";
+								string squery = "SELECT * FROM dats WHERE id=" + id;
 								SqliteCommand sslc = new SqliteCommand(squery, dbc);
+								SqliteDataReader ssldr = sslc.ExecuteReader();
+
+								// If the hash doesn't already exist, add it
+								if (!ssldr.HasRows)
+								{
+									squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
+									sslc = new SqliteCommand(squery, dbc);
 									sslc.ExecuteNonQuery();
+								}
+
+								ssldr.Dispose();
 								sslc.Dispose();
 							}
 
 							// If it doesn't exist, add the hash and the dat hash for a new id
 							else
 							{
-								string squery = "INSERT INTO data (key, value) VALUES (\"size\", \"" + rom.Size + "\")";
+								string squery = "INSERT INTO data (size, crc, md5, sha1, exists) VALUES"
+									+ " size=" + rom.Size + ","
+									+ " crc=\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+									+ " md5=\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+									+ " sha1=\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+									+ " exists=0)";
 								SqliteCommand sslc = new SqliteCommand(squery, dbc);
 								sslc.ExecuteNonQuery();
 
@@ -398,13 +413,10 @@ namespace SabreTools
 								sslc = new SqliteCommand(squery, dbc);
 								id = (long)sslc.ExecuteScalar();
 
-								squery = "INSERT INTO data (id, key, value) VALUES (\"" + id + "\", \"crc\", \"" + rom.CRC + "\"),"
+								squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
-									+ " (\"" + id + "\", \"md5\", \"" + rom.MD5 + "\"),"
-									+ " (\"" + id + "\", \"sha1\", \"" + rom.SHA1 + "\"),"
-									+ " (\"" + id + "\", \"dat\", \"" + key + "\"),"
-									+ " (\"" + id + "\", \"exists\", \"false\")";
 								sslc = new SqliteCommand(squery, dbc);
 								sslc.ExecuteNonQuery();
+								sslc.Dispose();
 							}
 						}
 					}
@@ -414,12 +426,14 @@ namespace SabreTools
 				// TODO: Remove orphaned files as well
 				foreach (string dathash in databaseDats)
 				{
-					query = "DELETE FROM data WHERE key=\"dat\" AND value=\"" + dathash + "\"";
+					query = "DELETE FROM dats WHERE hash=\"" + dathash + "\"";
 					slc = new SqliteCommand(query, dbc);
 					slc.ExecuteNonQuery();
 					slc.Dispose();
 				}
 			}
+
+			dbc.Dispose();
 		}
 
 		/// <summary>

RombaSharp/Partials/RombaSharp_Inits.cs

@@ -1,8 +1,8 @@
 using Mono.Data.Sqlite;
 using SabreTools.Helper;
-using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace SabreTools
 {
@@ -13,16 +13,102 @@ namespace SabreTools
 		/// <summary>
 		/// Wrap adding files to the depots
 		/// </summary>
-		/// <param name="inputs"></param>
+		/// <param name="inputs">List of input folders to use</param>
-		/// <param name="onlyNeeded"></param>
+		/// <param name="onlyNeeded">True if only files in the database and don't exist are added, false otherwise</param>
 		private static void InitArchive(List<string> inputs, bool onlyNeeded)
 		{
-			_logger.User("This feature is not yet implemented: archive");
+			// First we want to get just all directories from the inputs
+			List<string> onlyDirs = new List<string>();
+			foreach (string input in inputs)
+			{
+				if (Directory.Exists(input))
+				{
+					onlyDirs.Add(Path.GetFullPath(input));
+				}
+			}
 
-			// This should use the same thing as something like in SimpleSort, as in scan the archive and scan the files inside equally
+			// Then process all of the input directories into an internal DAT
-			// Either during or after, we then check against the database to see if there's any matches. If there's no match and
+			DatFile df = new DatFile();
-			// we say onlyNeeded, then don't add it. If there's no match and not onlyNeeded, add it with no DAT hash. If there's a match,
+			foreach (string dir in onlyDirs)
-			// and it doesn't say exists, add it and change the flag. If there's a match and says exists, skip it.
+			{
+				df.PopulateDatFromDir(dir, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger);
+			}
+
+			// Create an empty Dat for files that need to be rebuilt
+			DatFile need = new DatFile();
+			need.Files = new SortedDictionary<string, List<DatItem>>();
+
+			// Now that we have the Dats, add the files to the database
+			SqliteConnection dbc = new SqliteConnection(_connectionString);
+			foreach (string key in df.Files.Keys)
+			{
+				List<DatItem> datItems = df.Files[key];
+				foreach (Rom rom in datItems)
+				{
+					string query = "SELECT id FROM data WHERE size=" + rom.Size + " AND ("
+								+ "(crc=\"" + rom.CRC + "\" OR value=\"null\")"
+								+ " AND (md5=\"" + rom.MD5 + "\" OR value=\"null\")"
+								+ " AND (sha1=\"" + rom.SHA1 + "\" OR value=\"null\")"
+								+ " AND exists=0";
+					SqliteCommand slc = new SqliteCommand(query, dbc);
+					SqliteDataReader sldr = slc.ExecuteReader();
+
+					// If a row is returned, add the file and change the existence
+					if (sldr.HasRows)
+					{
+						sldr.Read();
+						string id = sldr.GetString(0);
+
+						string squery = "UPDATE data SET exists=1 WHERE id=" + id;
+						SqliteCommand sslc = new SqliteCommand(squery, dbc);
+						sslc.ExecuteNonQuery();
+						sslc.Dispose();
+
+						// Add the rom to the files that need to be rebuilt
+						if (need.Files.ContainsKey(key))
+						{
+							need.Files[key].Add(rom);
+						}
+						else
+						{
+							List<DatItem> temp = new List<DatItem>();
+							temp.Add(rom);
+							need.Files.Add(key, temp);
+						}
+					}
+
+					// If it doesn't exist, and we're not adding only needed files
+					else if (!onlyNeeded)
+					{
+						string squery = "INSERT INTO data (size, crc, md5, sha1, exists) VALUES"
+							+ " size=" + rom.Size + ","
+							+ " crc=\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+							+ " md5=\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+							+ " sha1=\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+							+ " exists=1)";
+						SqliteCommand sslc = new SqliteCommand(squery, dbc);
+						sslc.ExecuteNonQuery();
+						sslc.Dispose();
+
+						// Add the rom to the files that need to be rebuilt
+						if (need.Files.ContainsKey(key))
+						{
+							need.Files[key].Add(rom);
+						}
+						else
+						{
+							List<DatItem> temp = new List<DatItem>();
+							temp.Add(rom);
+							need.Files.Add(key, temp);
+						}
+					}
+				}
+			}
+
+			// Create the sorting object to use and rebuild the needed files
+			ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(0, 0, 0, 0);
+			SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, _logger);
+			ss.StartProcessing();
 		}
 
 		/// <summary>
@@ -99,40 +185,81 @@ namespace SabreTools
 		/// <summary>
 		/// Wrap looking up if hashes exist in the database
 		/// </summary>
-		/// <param name="inputs"></param>
+		/// <param name="inputs">List of input strings representing hashes to check for</param>
 		private static void InitLookup(List<string> inputs)
 		{
 			// First, try to figure out what type of hash each is by length and clean it
-			List<string> cleanedInputs = new List<string>();
+			List<string> crc = new List<string>();
+			List<string> md5 = new List<string>();
+			List<string> sha1 = new List<string>();
 			foreach (string input in inputs)
 			{
 				string temp = "";
 				if (input.Length == Constants.CRCLength)
 				{
 					temp = Style.CleanHashData(input, Constants.CRCLength);
+					if (temp != "")
+					{
+						crc.Add(temp);
+					}
 				}
 				else if (input.Length == Constants.MD5Length)
 				{
 					temp = Style.CleanHashData(input, Constants.MD5Length);
+					if (temp != "")
+					{
+						md5.Add(temp);
+					}
 				}
 				else if (input.Length == Constants.SHA1Length)
 				{
 					temp = Style.CleanHashData(input, Constants.SHA1Length);
-				}
-
-				// If the hash isn't empty, add it
 					if (temp != "")
 					{
-					cleanedInputs.Add(temp);
+						sha1.Add(temp);
+					}
 				}
 			}
 
 			// Now, search for each of them and return true or false for each
 			SqliteConnection dbc = new SqliteConnection(_connectionString);
-
+			foreach (string input in crc)
-			foreach (string input in cleanedInputs)
 			{
-				string query = "SELECT * FROM data WHERE value=\"" + input + "\"";
+				string query = "SELECT * FROM data WHERE crc=\"" + input + "\"";
+				SqliteCommand slc = new SqliteCommand(query, dbc);
+				SqliteDataReader sldr = slc.ExecuteReader();
+				if (sldr.HasRows)
+				{
+					_logger.User("For hash '" + input + "' there were " + sldr.RecordsAffected + " matches in the database");
+				}
+				else
+				{
+					_logger.User("Hash '" + input + "' had no matches in the database");
+				}
+
+				sldr.Dispose();
+				slc.Dispose();
+			}
+			foreach (string input in md5)
+			{
+				string query = "SELECT * FROM data WHERE md5=\"" + input + "\"";
+				SqliteCommand slc = new SqliteCommand(query, dbc);
+				SqliteDataReader sldr = slc.ExecuteReader();
+				if (sldr.HasRows)
+				{
+					_logger.User("For hash '" + input + "' there were " + sldr.RecordsAffected + " matches in the database");
+				}
+				else
+				{
+					_logger.User("Hash '" + input + "' had no matches in the database");
+				}
+
+				sldr.Dispose();
+				slc.Dispose();
+			}
+			foreach (string input in sha1)
+			{
+				string query = "SELECT * FROM data WHERE sha1=\"" + input + "\"";
 				SqliteCommand slc = new SqliteCommand(query, dbc);
 				SqliteDataReader sldr = slc.ExecuteReader();
 				if (sldr.HasRows)

SabreTools.Helper/Tools/DatabaseTools.cs

@@ -74,12 +74,24 @@ namespace SabreTools.Helper
 					string query = @"
 CREATE TABLE IF NOT EXISTS data (
 	'id'		INTEGER		NOT NULL
-	'key'	TEXT		NOT NULL
+	'size'		INTEGER		NOT NULL
-	'value'	TEXT		NOT NULL
+	'crc'		TEXT		NOT NULL
-	PRIMARY KEY (id, key, value)
+	'md5'		TEXT		NOT NULL
+	'sha1'		TEXT		NOT NULL
+	'exists'	INTEGER		NOT NULL
+	PRIMARY KEY (id)
 )";
 					SqliteCommand slc = new SqliteCommand(query, dbc);
 					slc.ExecuteNonQuery();
+
+					query = @"
+CREATE TABLE IF NOT EXISTS dats (
+	'id'	INTEGER		NOT NULL
+	'hash'	TEXT		NOT NULL
+	PRIMARY KEY (id, hash)
+)";
+					slc = new SqliteCommand(query, dbc);
+					slc.ExecuteNonQuery();
 					slc.Dispose();
 				}
 				else if (type == "headerer")