[DATabase] Miscellaneous integration cleanups

This cleans up a lot of things left over from the original integration of DATabaseTwo into DATabase. This also includes some updates that will make importing DATs even easier and make the import process more streamlined.

DATabase/DATabase.cs

@@ -43,9 +43,8 @@ namespace SabreTools
 			// Perform initial setup and verification
 			_logger = new Logger(true, "database.log");
 			_logger.Start();
-			DBTools.EnsureDatabase(_dbName, _connectionString);
-			Remapping.CreateRemappings();
 			Console.Clear();
+			Setup();
 
 			// Credits take precidence over all
 			if ((new List<string>(args)).Contains("--credits"))
@@ -370,7 +369,7 @@ namespace SabreTools
 
 			// If a switch that requires a filename is set and no file is, show the help screen
 			if (inputs.Count == 0 && ((convertMiss || romba) || convertCMP || convertRC || convertSD
-				|| convertXml || extsplit || hashsplit || import || (merge || diff) || stats || trim))
+				|| convertXml || extsplit || hashsplit || (merge || diff) || stats || trim))
 			{
 				_logger.Error("This feature requires at least one input");
 				Build.Help();
@@ -1309,7 +1308,7 @@ Make a selection:
 		private static void InitImport(bool ignore)
 		{
 			IImport imp = new ImportTwo(_datroot, _connectionString, _logger, ignore);
-			imp.ImportData();
+			imp.UpdateDatabase();
 		}
 
 		/// <summary>
@@ -1749,7 +1748,7 @@ Make a selection:
 		private static void Setup()
 		{
 			Remapping.CreateRemappings();
-			Build.Start("DATabaseTwo");
+			Build.Start("DATabase");
 
 			// Perform initial database and folder setup
 			if (!Directory.Exists(_datroot))

DATabase/ImportExport/Import.cs

@@ -34,7 +34,7 @@ namespace SabreTools
 		/// Import the data from file into the database
 		/// </summary>
 		/// <returns>True if the data was imported, false otherwise</returns>
-		public bool ImportData()
+		public bool UpdateDatabase()
 		{
 			// If file doesn't exist, error and return
 			if (!File.Exists(_filepath))

DATabase/ImportExport/ImportTwo.cs

@@ -36,199 +36,248 @@ namespace SabreTools
 		/// Perform initial or incremental import of DATs in the root folder
 		/// </summary>
 		/// <returns>True if the data could be inserted or updated correctly, false otherwise</returns>
-		public bool ImportData()
+		public bool UpdateDatabase()
 		{
 			_logger.User("Beginning import/update process");
+
+			Dictionary<Tuple<long, string, string>, int> missing = ImportData();
+			bool success = RemoveData(missing);
+
+			_logger.User("Import/update process complete!");
+
+			return success;
+		}
+
+		/// <summary>
+		/// Import data into the database and return all files not found in the list
+		/// </summary>
+		/// <returns>List of files that were not found in the audit</returns>
+		private Dictionary<Tuple<long, string, string>, int> ImportData()
+		{
+			// Create the empty dictionary for file filtering and output
+			Dictionary<Tuple<long, string, string>, int> dbfiles = new Dictionary<Tuple<long, string, string>, int>();
+
 			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
 			{
 				dbc.Open();
+				_logger.User("Populating reference objects");
 
-				Dictionary<string, int> hashes = new Dictionary<string, int>();
-				string query = "SELECT sha1, id FROM dats";
+				// Populate the list of files in the database with Tuples (size, sha1, name)
+				string query = "SELECT id, size, sha1, name FROM dats";
 				using (SqliteCommand slc = new SqliteCommand(query, dbc))
 				{
 					using (SqliteDataReader sldr = slc.ExecuteReader())
 					{
 						while (sldr.Read())
 						{
-							hashes.Add(sldr.GetString(0), sldr.GetInt32(1));
+							dbfiles.Add(Tuple.Create(sldr.GetInt64(1), sldr.GetString(2), sldr.GetString(3)), sldr.GetInt32(0));
 						}
 					}
 				}
 
-				SHA1 sha1 = SHA1.Create();
-				query = "SELECT * FROM system";
+				// Populate the list of systems
+				Dictionary<string, int> systems = new Dictionary<string, int>();
+				query = "SELECT id, manufacturer, name FROM system";
 				using (SqliteCommand slc = new SqliteCommand(query, dbc))
 				{
 					using (SqliteDataReader sldr = slc.ExecuteReader())
 					{
 						while (sldr.Read())
 						{
-							int systemid = sldr.GetInt32(0);
-							string system = _datroot + Path.DirectorySeparatorChar + sldr.GetString(1) + " - " + sldr.GetString(2);
-							system = system.Trim();
+							systems.Add(sldr.GetString(1) + " - " + sldr.GetString(2), sldr.GetInt32(0));
+						}
+					}
+				}
 
-							_logger.User("System: " + system.Remove(0, 5));
+				// Populate the list of sources (initial)
+				SortedDictionary<string, int> sources = new SortedDictionary<string, int>();
+				sources.Add("default", 0);
+				query = "SELECT name, id FROM source";
+				using (SqliteCommand sslc = new SqliteCommand(query, dbc))
+				{
+					using (SqliteDataReader ssldr = sslc.ExecuteReader())
+					{
+						while (ssldr.Read())
+						{
+							sources.Add(ssldr.GetString(0).ToLowerInvariant(), ssldr.GetInt32(1));
+						}
+					}
+				}
 
-							// Audit all DATs in the folder
-							foreach (string file in Directory.GetFiles(system, "*", SearchOption.AllDirectories))
+				// Interate through each system and check files
+				SHA1 sha1 = SHA1.Create();
+				foreach (KeyValuePair<string, int> kv in systems)
+				{
+					_logger.User("Processing DATs for system: '" + kv.Key + "'");
+
+					// Set the folder to iterate through based on the DAT root
+					string folder = _datroot + Path.DirectorySeparatorChar + kv.Key.Trim();
+
+					// Audit all files in the folder
+					foreach (string file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories))
+					{
+						// First get the file information for comparison
+						long size = (new FileInfo(file)).Length;
+						string hash = "";
+						using (FileStream fs = File.Open(file, FileMode.Open))
+						{
+							hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", "");
+						}
+
+						// If it's not in the list of known files, add it
+						if (!dbfiles.ContainsKey(Tuple.Create(size, hash, file)))
+						{
+							// First add the information to the database as is and return the new insert ID
+							_logger.Log("Adding file information for " + Path.GetFileName(file));
+
+							int hashid = -1;
+							query = @"INSERT INTO dats (size, sha1, name)
+VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"')";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
 							{
-								string hash = "";
-								using (FileStream fs = File.Open(file, FileMode.Open))
+								slc.ExecuteNonQuery();
+							}
+
+							//query = "SELECT last_insertConstants.Rowid()";
+							query = "SELECT last_insert_rowid()";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
+							{
+								using (SqliteDataReader sldr = slc.ExecuteReader())
 								{
-									hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", "");
-								}
-
-								// If the hash isn't in add it and all required information
-								int hashid = -1;
-								if (!hashes.ContainsKey(hash))
-								{
-									_logger.Log("Adding file information for " + Path.GetFileName(file));
-
-									string squery = @"BEGIN;
-INSERT INTO dats (size, sha1, name)
-VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"');
-SELECT last_insertConstants.Rowid();
-COMMIT;";
-									using (SqliteCommand sslc = new SqliteCommand(squery, dbc))
+									if (sldr.Read())
 									{
-										using (SqliteDataReader ssldr = sslc.ExecuteReader())
-										{
-											if (ssldr.Read())
-											{
-												hashid = ssldr.GetInt32(0);
-											}
-										}
-									}
-
-									// Add the hash to the temporary Dictionary
-									hashes.Add(hash, hashid);
-
-									// Now try to determine the source for the file based on the name
-									string source = GetSourceFromFileName(Path.GetFileName(file));
-									int sourceid = 0;
-
-									SortedDictionary<string, int> sources = new SortedDictionary<string, int>();
-									sources.Add("default", 0);
-									query = "SELECT name, id FROM source";
-									using (SqliteCommand sslc = new SqliteCommand(query, dbc))
-									{
-										using (SqliteDataReader ssldr = sslc.ExecuteReader())
-										{
-											while (ssldr.Read())
-											{
-												sources.Add(ssldr.GetString(0).ToLowerInvariant(), ssldr.GetInt32(1));
-											}
-										}
-									}
-
-									// Only if we're not ignoring new sources should be ask the user for input
-									if (!_ignore)
-									{
-										// We want to reset "Default" at this point, just in case
-										if (source.ToLowerInvariant() == "default")
-										{
-											source = "";
-										}
-
-										// If the source is blank, ask the user to supply one
-										while (source == "" && sourceid == 0)
-										{
-											Console.Clear();
-											Build.Start("DATabaseTwo");
-
-											Console.WriteLine("Sources:");
-											foreach (KeyValuePair<string, int> pair in sources)
-											{
-												Console.WriteLine("    " + pair.Value + " - " + pair.Key);
-											}
-											Console.WriteLine("\nFor file name: " + Path.GetFileName(file));
-											Console.Write("Select a source above or enter a new one: ");
-											source = Console.ReadLine();
-
-											Int32.TryParse(source, out sourceid);
-
-											// If the value could be parsed, reset the source string
-											if (sourceid != 0)
-											{
-												source = "";
-											}
-
-											// If the source ID is set check to see if it's valid
-											if (sourceid != 0 && !sources.ContainsValue(sourceid))
-											{
-												Console.WriteLine("Invalid selection: " + sourceid);
-												Console.ReadLine();
-												sourceid = 0;
-											}
-										}
-
-										// If the source isn't in, add it and get the insert id
-										if (source != "" && sourceid == 0 && !sources.ContainsKey(source.ToLowerInvariant()))
-										{
-											string tquery = @"BEGIN;
-INSERT INTO source (name, url)
-VALUES ('" + source + @"', '');
-SELECT last_insertConstants.Rowid();
-COMMIT;";
-											using (SqliteCommand sslc = new SqliteCommand(tquery, dbc))
-											{
-												using (SqliteDataReader ssldr = sslc.ExecuteReader())
-												{
-													if (ssldr.Read())
-													{
-														sourceid = ssldr.GetInt32(0);
-													}
-												}
-											}
-
-											// Add the new source to the temporary Dictionary
-											sources.Add(source.ToLowerInvariant(), sourceid);
-										}
-										// Otherwise, get the ID
-										else if (source != "" && sourceid == 0 && sources.ContainsKey(source.ToLowerInvariant()))
-										{
-											try
-											{
-												sourceid = sources[source.ToLowerInvariant()];
-											}
-											catch
-											{
-												sourceid = 0;
-											}
-										}
-										// Otherwise, we should already have an ID
-									}
-									else
-									{
-										try
-										{
-											sourceid = sources[source.ToLowerInvariant()];
-										}
-										catch
-										{
-											sourceid = 0;
-										}
-									}
-
-									// Add the source and system link to the database
-									string uquery = @"INSERT OR IGNORE INTO datsdata (id, key, value)
-VALUES (" + hashid + ", 'source', '" + sourceid + @"'),
-(" + hashid + ", 'system', '" + systemid + "')";
-									using (SqliteCommand uslc = new SqliteCommand(uquery, dbc))
-									{
-										uslc.ExecuteNonQuery();
+										hashid = sldr.GetInt32(0);
 									}
 								}
 							}
+
+							// Next we try to figure out the source ID from the name
+							string possiblesource = GetSourceFromFileName(Path.GetFileName(file));
+
+							// Try to get the source ID from the name
+							int sourceid = (sources.ContainsKey(possiblesource.ToLowerInvariant()) ? sources[possiblesource] : 0);
+
+							// If we have the "default" ID and we're not ignoring new sources, prompt for a source input
+							if (!_ignore)
+							{
+								// We want to reset "Default" at this point, just in case
+								if (possiblesource.ToLowerInvariant() == "default")
+								{
+									possiblesource = "";
+								}
+
+								// If the source is blank, ask the user to supply one
+								while (possiblesource == "" && sourceid == 0)
+								{
+									Console.Clear();
+									Build.Start("DATabaseTwo");
+
+									Console.WriteLine("Sources:");
+									foreach (KeyValuePair<string, int> pair in sources)
+									{
+										Console.WriteLine("    " + pair.Value + " - " + Style.SentenceCase(pair.Key));
+									}
+									Console.WriteLine("\nFor file name: " + Path.GetFileName(file));
+									Console.Write("Select a source above or enter a new one: ");
+									possiblesource = Console.ReadLine();
+
+									Int32.TryParse(possiblesource, out sourceid);
+
+									// If the value could be parsed, reset the source string
+									if (sourceid != 0)
+									{
+										possiblesource = "";
+									}
+
+									// If the source ID is set check to see if it's valid
+									if (sourceid != 0 && !sources.ContainsValue(sourceid))
+									{
+										Console.WriteLine("Invalid selection: " + sourceid);
+										Console.ReadLine();
+										sourceid = 0;
+									}
+								}
+
+								// If we have a non-empty possible source and it's in the database, get the id
+								if (possiblesource != "" && sources.ContainsKey(possiblesource.ToLowerInvariant()))
+								{
+									sourceid = sources[possiblesource.ToLowerInvariant()];
+								}
+
+								// If we have a non-empty possible source and it's not in the database, insert and get the id
+								else if (possiblesource != "" && !sources.ContainsKey(possiblesource.ToLowerInvariant()))
+								{
+									query = @"BEGIN;
+INSERT INTO source (name, url)
+VALUES ('" + possiblesource + @"', '');
+SELECT last_insertConstants.Rowid();
+COMMIT;";
+									using (SqliteCommand slc = new SqliteCommand(query, dbc))
+									{
+										using (SqliteDataReader sldr = slc.ExecuteReader())
+										{
+											if (sldr.Read())
+											{
+												sourceid = sldr.GetInt32(0);
+											}
+										}
+									}
+
+									// Add the new source to the current dictionary
+									sources.Add(possiblesource.ToLowerInvariant(), sourceid);
+								}
+							}
+
+							// Now that we have a source ID, we can add the mappings for system and source to the database
+							query = @"INSERT OR IGNORE INTO datsdata (id, key, value)
+VALUES (" + hashid + ", 'source', '" + sourceid + @"'),
+(" + hashid + ", 'system', '" + kv.Value + "')";
+							using (SqliteCommand slc = new SqliteCommand(query, dbc))
+							{
+								slc.ExecuteNonQuery();
+							}
+						}
+
+						// Otherwise, remove it from the list of found items
+						else
+						{
+							dbfiles.Remove(Tuple.Create(size, hash, file));
 						}
 					}
 				}
 			}
 
-			_logger.User("Import/update process complete!");
+			return dbfiles;
+		}
 
-			return true;
+		/// <summary>
+		/// Remove all data associated with various files
+		/// </summary>
+		/// <param name="missing">List of file identifiers to remove from the database</param>
+		/// <returns>True if everything went well, false otherwise</returns>
+		private bool RemoveData(Dictionary<Tuple<long, string, string>, int> missing)
+		{
+			bool success = true;
+
+			using (SqliteConnection dbc = new SqliteConnection(_connectionString))
+			{
+				dbc.Open();
+
+				// Get a comma-separated list of IDs from the input files
+				string idlist = String.Join(",", missing.Values);
+
+				// Now remove all of the files from the database
+				string query = @"BEGIN;
+DELETE FROM datsdata WHERE id IN (" + idlist + @");
+DELETE FROM dats WHERE id IN (" + idlist + @");
+COMMIT;";
+				using (SqliteCommand slc = new SqliteCommand(query, dbc))
+				{
+					slc.ExecuteNonQuery();
+				}
+			}
+
+			return success;
 		}
 
 		/// <summary>
@@ -276,6 +325,12 @@ VALUES (" + hashid + ", 'source', '" + sourceid + @"'),
 			else if (Regex.IsMatch(filename, Constants.GoodXmlPattern))
 			{
 				fileinfo = Regex.Match(filename, Constants.GoodXmlPattern).Groups;
+				if (!Remapping.Good.ContainsKey(fileinfo[1].Value))
+				{
+					_logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped.");
+					return source;
+				}
+				source = "Good";
 			}
 			else if (Regex.IsMatch(filename, Constants.MaybeIntroPattern))
 			{

README.MD

@@ -19,7 +19,7 @@ A bare-bones attempt at providing a true GUI experience for the SabreTools suite
 <p/>
 The main tool of the SabreTools suite. Performs the majority of the core features of the parent project, including the following:
 <ul>
-	<li>Importing and Generating DAT files in ClrMamePro and XML formats</li>
+	<li>Importing and Generating DAT files in ClrMamePro and XML formats (current version formerly <i>DATabaseTwo</i>)</li>
 	<li>Converting DATs from any format to ClrMamePro, Logiqx XML, SabreDAT XML, and to missfile (last part formerly <i>DatToMiss</i>; requested by Obiwantje)</li>
 	<li>Trim DAT entries and optionally merge into a single game (formerly <i>SingleGame</i>; requested by Kludge)</li>
 	<li>Split a DAT using two different file extensions within the DAT (formerly <i>DatSplit</i>)</li>
@@ -39,10 +39,6 @@ The main tool of the SabreTools suite. Performs the majority of the core feature
 </ul>
 This tool can both be used with command line parameters or with the built-in text menu system.
 
-<b>DATabaseTwo</b>
-<p/>
-An in-progress replacement for the Import and Generate functions of <i>DATabase</i>. This is the recommended version to use for mass DAT management but it is still lacking some advanced features found only in the <i>DATabase</i> version.
-
 <b>Headerer</b>
 <p/>
 A simple auxilary tool that detects and removes headers and also restores headers for the following systems:

SabreHelper/Data/Constants.cs

@@ -20,16 +20,16 @@
 		public static string GoodXmlPattern = @"^(Good.*?)_.*\.xml";
 		public static string MamePattern = @"^(.*)\.xml$";
 		public static string MaybeIntroPattern = @"(.*?) \[T-En\].*\((\d{8})\)\.dat$";
-		public static string NoIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\)\.dat$";
-		public static string NoIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).dat";
-		public static string NoIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\)\.dat";
+		public static string NoIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\).*\.dat$";
+		public static string NoIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).*.dat";
+		public static string NoIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\).*\.dat";
 		public static string NonGoodPattern = @"^(NonGood.*?)( .*?)?.xml";
 		public static string NonGoodSpecialPattern = @"^(NonGood.*?)( .*)?.dat";
 		public static string RedumpPattern = @"^(.*?) \((\d{8} \d{2}-\d{2}-\d{2})\)\.dat$";
 		public static string RedumpBiosPattern = @"^(.*?) \(\d+\) \((\d{4}-\d{2}-\d{2})\)\.dat$";
-		public static string TosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$";
-		public static string TosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$";
-		public static string TosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$";
+		public static string TosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$";
+		public static string TosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$";
+		public static string TosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$";
 		public static string TruripPattern = @"^(.*) - .* \(trurip_XML\)\.dat$";
 		public static string ZandroPattern = @"^SMW-.*.xml";
 

SabreHelper/Interfaces/IImport.cs

@@ -2,6 +2,6 @@
 {
 	public interface IImport
 	{
-		bool ImportData();
+		bool UpdateDatabase();
 	}
 }

SabreHelper/Mappings/NoIntro.xml

@@ -38,6 +38,7 @@
 	<mapping from="Nintendo - Game Boy" to="Nintendo - Game Boy, Game Boy Color" />
 	<mapping from="Nintendo - Game Boy Advance" to="Nintendo - Game Boy Advance" />
 	<mapping from="Nintendo - Game Boy Advance (e-Cards)" to="Nintendo - Game Boy Advance" />
+	<mapping from="Nintendo - Game Boy Advance MultiBoots" to="Nintendo - Game Boy Advance" />
 	<mapping from="Nintendo - Game Boy Color" to="Nintendo - Game Boy, Game Boy Color" />
 	<mapping from="Nintendo - New Nintendo 3DS" to="Nintendo - Nintendo 3DS, New Nintendo 3DS" />
 	<mapping from="Nintendo - New Nintendo 3DS (DLC)" to="Nintendo - Nintendo 3DS, New Nintendo 3DS" />

SabreHelper/Style.cs

@@ -218,5 +218,22 @@ namespace SabreTools.Helper
 			// Return formatted number with suffix
 			return readable.ToString("0.### ") + suffix;
 		}
+
+		/// <summary>
+		/// Converts a string to sentence case.
+		/// </summary>
+		/// <param name="input">The string to convert.</param>
+		/// <returns>A string representing a sentence case string</returns>
+		/// <remarks>http://stackoverflow.com/questions/3141426/net-method-to-convert-a-string-to-sentence-case</remarks>
+		public static string SentenceCase(string input)
+		{
+			if (input.Length < 1)
+			{
+				return input;
+			}
+
+			string sentence = input.ToLower();
+			return sentence[0].ToString().ToUpper() + sentence.Substring(1);
+		}
 	}
 }