[RombaSharp] Implement database based directly on Romba KV implementation

RombaSharp/Partials/RombaSharp_Helpers.cs

@@ -232,43 +232,23 @@ namespace SabreTools
 			SqliteConnection dbc = new SqliteConnection(_connectionString);
 			dbc.Open();
 
-			// Total uncompressed size
-			string query = "SELECT SUM(size) FROM data";
-			SqliteCommand slc = new SqliteCommand(query, dbc);
-			_logger.User("Uncompressed size: " + Style.GetBytesReadable((long)slc.ExecuteScalar()));
-
-			// Total number of files
-			query = "SELECT COUNT(*) FROM data";
-			slc = new SqliteCommand(query, dbc);
-			_logger.User("Total files: " + (long)slc.ExecuteScalar());
-
-			// Total number of files that exist
-			query = "SELECT COUNT(*) FROM data WHERE indepot=1";
-			slc = new SqliteCommand(query, dbc);
-			_logger.User("Total files in depots: " + (long)slc.ExecuteScalar());
-
-			// Total number of files that are missing
-			query = "SELECT COUNT(*) FROM data WHERE indepot=0";
-			slc = new SqliteCommand(query, dbc);
-			_logger.User("Total files missing: " + (long)slc.ExecuteScalar());
-
 			// Total number of CRCs
-			query = "SELECT COUNT(crc) FROM data WHERE NOT crc=\"null\"";
+			string query = "SELECT COUNT(*) FROM crc";
-			slc = new SqliteCommand(query, dbc);
+			SqliteCommand slc = new SqliteCommand(query, dbc);
 			_logger.User("Total CRCs: " + (long)slc.ExecuteScalar());
 
 			// Total number of MD5s
-			query = "SELECT COUNT(md5) FROM data WHERE NOT md5=\"null\"";
+			query = "SELECT COUNT(*) FROM md5";
 			slc = new SqliteCommand(query, dbc);
 			_logger.User("Total MD5s: " + (long)slc.ExecuteScalar());
 
 			// Total number of SHA1s
-			query = "SELECT COUNT(sha1) FROM data WHERE NOT sha1=\"null\"";
+			query = "SELECT COUNT(*) FROM sha1";
 			slc = new SqliteCommand(query, dbc);
 			_logger.User("Total SHA1s: " + (long)slc.ExecuteScalar());
 
 			// Total number of DATs
-			query = "SELECT COUNT(*) FROM dats GROUP BY hash";
+			query = "SELECT COUNT(*) FROM dat";
 			slc = new SqliteCommand(query, dbc);
 			_logger.User("Total DATs: " + (long)slc.ExecuteScalar());
 
@@ -298,6 +278,7 @@ namespace SabreTools
 		/// <summary>
 		/// Export the current database to CSV
 		/// </summary>
+		/// <remarks>REDO</remarks>
 		private static void ExportDatabase()
 		{
 			SqliteConnection dbc = new SqliteConnection(_connectionString);
@@ -350,7 +331,7 @@ namespace SabreTools
 		/// <summary>
 		/// Populate or refresh the database information
 		/// </summary>
-		/// <remarks>Each hash has the following attributes: size, crc, md5, sha-1, dathash, indepot</remarks>
+		/// <remarks>This has no link between Dat and file at all...</remarks>
 		private static void RefreshDatabase()
 		{
 			// Make sure the db is set
@@ -388,7 +369,7 @@ namespace SabreTools
 			_logger.User("Populating the list of existing DATs");
 			DateTime start = DateTime.Now;
 
-			string query = "SELECT DISTINCT hash FROM dats";
+			string query = "SELECT DISTINCT hash FROM dat";
 			SqliteCommand slc = new SqliteCommand(query, dbc);
 			SqliteDataReader sldr = slc.ExecuteReader();
 			if (sldr.HasRows)
@@ -444,6 +425,12 @@ namespace SabreTools
 				// If the Dat wasn't empty, add the information
 				if (tempdat.Files.Count != 0)
 				{
+					string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
+					string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
+					string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
+					string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
+					string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
+
 					// Loop through the parsed entries
 					foreach (string romkey in tempdat.Files.Keys)
 					{
@@ -451,60 +438,55 @@ namespace SabreTools
 						{
 							_logger.Verbose("Checking and adding file '" + rom.Name);
 
-							query = "SELECT id FROM data WHERE size=" + rom.Size + " AND ("
+							if (!String.IsNullOrEmpty(rom.CRC))
-								+ "(crc=\"" + rom.CRC + "\" OR crc=\"null\")"
-								+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
-								+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\"))";
-							slc = new SqliteCommand(query, dbc);
-							sldr = slc.ExecuteReader();
-								
-							// If the hash exists in the database, add the dat hash for that id if needed
-							if (sldr.HasRows)
 							{
-								sldr.Read();
+								crcquery += " (\"" + rom.CRC + "\"),";
-								long id = sldr.GetInt64(0);
-
-								string squery = "SELECT * FROM dats WHERE id=" + id;
-								SqliteCommand sslc = new SqliteCommand(squery, dbc);
-								SqliteDataReader ssldr = sslc.ExecuteReader();
-
-								// If the hash doesn't already exist, add it
-								if (!ssldr.HasRows)
-								{
-									squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
-									sslc = new SqliteCommand(squery, dbc);
-									sslc.ExecuteNonQuery();
-								}
-
-								ssldr.Dispose();
-								sslc.Dispose();
 							}
-
+							if (!String.IsNullOrEmpty(rom.MD5))
-							// If it doesn't exist, add the hash and the dat hash for a new id
-							else
 							{
-								string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+								md5query += " (\"" + rom.MD5 + "\"),";
-									+ rom.Size + ","
+							}
-									+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+							if (!String.IsNullOrEmpty(rom.SHA1))
-									+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+							{
-									+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+								sha1query += " (\"" + rom.SHA1 + "\"),";
-									+ "0)";
+							}
-								SqliteCommand sslc = new SqliteCommand(squery, dbc);
+							if (!String.IsNullOrEmpty(rom.CRC) && !String.IsNullOrEmpty(rom.SHA1))
-								sslc.ExecuteNonQuery();
+							{
-
+								crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),";
-								long id = -1;
+							}
-
+							if (!String.IsNullOrEmpty(rom.MD5) && !String.IsNullOrEmpty(rom.SHA1))
-								squery = @"select last_insert_rowid()";
+							{
-								sslc = new SqliteCommand(squery, dbc);
+								md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),";
-								id = (long)sslc.ExecuteScalar();
-
-								squery = "INSERT INTO dats (id, hash) VALUES (\"" + id + "\", \"" + key + "\")";
-								sslc = new SqliteCommand(squery, dbc);
-								sslc.ExecuteNonQuery();
-								sslc.Dispose();
 							}
 						}
 					}
+
+					// Now run the queries after fixing them
+					if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
+					{
+						slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
+						slc.ExecuteNonQuery();
+					}
+					if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
+					{
+						slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
+						slc.ExecuteNonQuery();
+					}
+					if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
+					{
+						slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
+						slc.ExecuteNonQuery();
+					}
+					if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
+					{
+						slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
+						slc.ExecuteNonQuery();
+					}
+					if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
+					{
+						slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
+						slc.ExecuteNonQuery();
+					}
 				}
 			}
 			_logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));

RombaSharp/Partials/RombaSharp_Inits.cs

@@ -47,50 +47,54 @@ namespace SabreTools
 				List<DatItem> datItems = df.Files[key];
 				foreach (Rom rom in datItems)
 				{
-					string query = "SELECT id FROM data WHERE size=" + rom.Size
+					bool add = false;
-								+ " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")"
+					if (onlyNeeded)
-								+ " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")"
-								+ " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")"
-								+ " AND indepot=0";
-					SqliteCommand slc = new SqliteCommand(query, dbc);
-					SqliteDataReader sldr = slc.ExecuteReader();
-
-					// If a row is returned, add the file and change the existence
-					if (sldr.HasRows)
 					{
-						sldr.Read();
+						string query = @"
-						long id = sldr.GetInt64(0);
+SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1
+	JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1
+WHERE crcsha1.crc=""" + rom.CRC + @"""
+	OR md5sha1.md5=""" + rom.MD5 + @"""
+	OR md5sha1.sha1=""" + rom.SHA1 + "\"";
+						SqliteCommand slc = new SqliteCommand(query, dbc);
+						SqliteDataReader sldr = slc.ExecuteReader();
 
-						string squery = "UPDATE data SET indepot=1 WHERE id=" + id;
+						// If a row is returned, add the file and change the existence
-						SqliteCommand sslc = new SqliteCommand(squery, dbc);
+						if (sldr.HasRows)
-						sslc.ExecuteNonQuery();
-						sslc.Dispose();
-
-						// Add the rom to the files that need to be rebuilt
-						if (need.Files.ContainsKey(key))
 						{
-							need.Files[key].Add(rom);
+							add = true;
-						}
-						else
-						{
-							List<DatItem> temp = new List<DatItem>();
-							temp.Add(rom);
-							need.Files.Add(key, temp);
 						}
 					}
-
+					else
-					// If it doesn't exist, and we're not adding only needed files
-					else if (!onlyNeeded)
 					{
-						string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES ("
+						add = true;
-							+ rom.Size + ","
+					}
-							+ "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\","
+					
-							+ "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\","
+					// If we're supposed to add the file to the depot, add it to the list
-							+ "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\","
+					if (add)
-							+ "1)";
+					{
-						SqliteCommand sslc = new SqliteCommand(squery, dbc);
+						_logger.User("Adding file \"" + rom.Name + "\" to the database");
-						sslc.ExecuteNonQuery();
+
-						sslc.Dispose();
+						// Insert new or updated information into the database
+						string query = "INSERT OR IGNORE INTO crc (crc) VALUES (\"" + rom.CRC + "\")";
+						SqliteCommand slc = new SqliteCommand(query, dbc);
+						slc.ExecuteNonQuery();
+
+						query = "INSERT OR IGNORE INTO md5 (md5) VALUES (\"" + rom.MD5 + "\")";
+						slc = new SqliteCommand(query, dbc);
+						slc.ExecuteNonQuery();
+
+						query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES (\"" + rom.SHA1 + "\")";
+						slc = new SqliteCommand(query, dbc);
+						slc.ExecuteNonQuery();
+
+						query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\")";
+						slc = new SqliteCommand(query, dbc);
+						slc.ExecuteNonQuery();
+
+						query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\")";
+						slc = new SqliteCommand(query, dbc);
+						slc.ExecuteNonQuery();
 
 						// Add the rom to the files that need to be rebuilt
 						if (need.Files.ContainsKey(key))
@@ -148,57 +152,12 @@ namespace SabreTools
 					Directory.CreateDirectory(outputFolder);
 				}
 
-				// Then get all hashes associated with this DAT
+				// Get the depots that are online
-				string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\"";
+				List<string> onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList();
-				SqliteCommand slc = new SqliteCommand(query, dbc);
-				SqliteDataReader sldr = slc.ExecuteReader();
-				if (sldr.HasRows)
-				{
-					while (sldr.Read())
-					{
-						string sha1 = sldr.GetString(0);
-						string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz");
 
-						// Find the first depot that contains the folder
+				// Loop over the depots and rebuild as needed
-						foreach (string depot in _depots.Keys)
+				SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, copy, copy, asl, false, _logger);
-						{
+				ss.StartProcessing();
-							// If the depot is online, check it
-							if (_depots[depot].Item2)
-							{
-								if (File.Exists(Path.Combine(depot, filename)))
-								{
-									if (copy)
-									{
-										if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename))))
-										{
-											Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename)));
-										}
-
-										try
-										{
-											File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true);
-										}
-										catch { }
-									}
-									else
-									{
-										ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger);
-									}
-									continue;
-								}
-							}
-						}
-					}
-				}
-
-				// Now that we have extracted everything, we rebuild to the output folder
-				if (!copy)
-				{
-					List<string> temp = new List<string>();
-					temp.Add(_tmpdir);
-					SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, true, false, false, asl, false, _logger);
-					ss.StartProcessing();
-				}
 			}
 
 			dbc.Dispose();
@@ -305,7 +264,7 @@ namespace SabreTools
 			// Now, search for each of them and return true or false for each
 			foreach (string input in crc)
 			{
-				string query = "SELECT * FROM data WHERE crc=\"" + input + "\"";
+				string query = "SELECT * FROM crc WHERE crc=\"" + input + "\"";
 				SqliteCommand slc = new SqliteCommand(query, dbc);
 				SqliteDataReader sldr = slc.ExecuteReader();
 				if (sldr.HasRows)
@@ -322,7 +281,7 @@ namespace SabreTools
 			}
 			foreach (string input in md5)
 			{
-				string query = "SELECT * FROM data WHERE md5=\"" + input + "\"";
+				string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\"";
 				SqliteCommand slc = new SqliteCommand(query, dbc);
 				SqliteDataReader sldr = slc.ExecuteReader();
 				if (sldr.HasRows)
@@ -339,7 +298,7 @@ namespace SabreTools
 			}
 			foreach (string input in sha1)
 			{
-				string query = "SELECT * FROM data WHERE sha1=\"" + input + "\"";
+				string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\"";
 				SqliteCommand slc = new SqliteCommand(query, dbc);
 				SqliteDataReader sldr = slc.ExecuteReader();
 				if (sldr.HasRows)

SabreTools.Helper/Tools/DatabaseTools.cs

@@ -72,23 +72,51 @@ namespace SabreTools.Helper
 				if (type == "rombasharp")
 				{
 					string query = @"
-CREATE TABLE IF NOT EXISTS data (
+CREATE TABLE IF NOT EXISTS crc (
-	'id'		INTEGER		NOT NULL,
+	'crc'	TEXT		NOT NULL,
-	'size'		INTEGER		NOT NULL,
+	PRIMARY KEY (crc)
-	'crc'		TEXT		NOT NULL,
-	'md5'		TEXT		NOT NULL,
-	'sha1'		TEXT		NOT NULL,
-	'indepot'	INTEGER		NOT NULL,
-	PRIMARY KEY (id)
 )";
 					SqliteCommand slc = new SqliteCommand(query, dbc);
 					slc.ExecuteNonQuery();
 
 					query = @"
-CREATE TABLE IF NOT EXISTS dats (
+CREATE TABLE IF NOT EXISTS md5 (
-	'id'	INTEGER		NOT NULL,
+	'md5'	TEXT		NOT NULL,
+	PRIMARY KEY (md5)
+)";
+					slc = new SqliteCommand(query, dbc);
+					slc.ExecuteNonQuery();
+
+					query = @"
+CREATE TABLE IF NOT EXISTS sha1 (
+	'sha1'	TEXT		NOT NULL,
+	PRIMARY KEY (sha1)
+)";
+					slc = new SqliteCommand(query, dbc);
+					slc.ExecuteNonQuery();
+
+					query = @"
+CREATE TABLE IF NOT EXISTS crcsha1 (
+	'crc'	TEXT		NOT NULL,
+	'sha1'	TEXT		NOT NULL,
+	PRIMARY KEY (crc, sha1)
+)";
+					slc = new SqliteCommand(query, dbc);
+					slc.ExecuteNonQuery();
+
+					query = @"
+CREATE TABLE IF NOT EXISTS md5sha1 (
+	'md5'	TEXT		NOT NULL,
+	'sha1'	TEXT		NOT NULL,
+	PRIMARY KEY (md5, sha1)
+)";
+					slc = new SqliteCommand(query, dbc);
+					slc.ExecuteNonQuery();
+
+					query = @"
+CREATE TABLE IF NOT EXISTS dat (
 	'hash'	TEXT		NOT NULL,
-	PRIMARY KEY (id, hash)
+	PRIMARY KEY (hash)
 )";
 					slc = new SqliteCommand(query, dbc);
 					slc.ExecuteNonQuery();