diff --git a/DATabase/DATabase.cs b/DATabase/DATabase.cs index 5488acbe..dc29864b 100644 --- a/DATabase/DATabase.cs +++ b/DATabase/DATabase.cs @@ -43,9 +43,8 @@ namespace SabreTools // Perform initial setup and verification _logger = new Logger(true, "database.log"); _logger.Start(); - DBTools.EnsureDatabase(_dbName, _connectionString); - Remapping.CreateRemappings(); Console.Clear(); + Setup(); // Credits take precidence over all if ((new List(args)).Contains("--credits")) @@ -370,7 +369,7 @@ namespace SabreTools // If a switch that requires a filename is set and no file is, show the help screen if (inputs.Count == 0 && ((convertMiss || romba) || convertCMP || convertRC || convertSD - || convertXml || extsplit || hashsplit || import || (merge || diff) || stats || trim)) + || convertXml || extsplit || hashsplit || (merge || diff) || stats || trim)) { _logger.Error("This feature requires at least one input"); Build.Help(); @@ -1309,7 +1308,7 @@ Make a selection: private static void InitImport(bool ignore) { IImport imp = new ImportTwo(_datroot, _connectionString, _logger, ignore); - imp.ImportData(); + imp.UpdateDatabase(); } /// @@ -1749,7 +1748,7 @@ Make a selection: private static void Setup() { Remapping.CreateRemappings(); - Build.Start("DATabaseTwo"); + Build.Start("DATabase"); // Perform initial database and folder setup if (!Directory.Exists(_datroot)) diff --git a/DATabase/ImportExport/Import.cs b/DATabase/ImportExport/Import.cs index fe7b778c..04f7a23d 100644 --- a/DATabase/ImportExport/Import.cs +++ b/DATabase/ImportExport/Import.cs @@ -34,7 +34,7 @@ namespace SabreTools /// Import the data from file into the database /// /// True if the data was imported, false otherwise - public bool ImportData() + public bool UpdateDatabase() { // If file doesn't exist, error and return if (!File.Exists(_filepath)) diff --git a/DATabase/ImportExport/ImportTwo.cs b/DATabase/ImportExport/ImportTwo.cs index 42c7b9c2..3f56f890 100644 --- a/DATabase/ImportExport/ImportTwo.cs +++ b/DATabase/ImportExport/ImportTwo.cs @@ -36,199 +36,248 @@ namespace SabreTools /// Perform initial or incremental import of DATs in the root folder /// /// True if the data could be inserted or updated correctly, false otherwise - public bool ImportData() + public bool UpdateDatabase() { _logger.User("Beginning import/update process"); + + Dictionary, int> missing = ImportData(); + bool success = RemoveData(missing); + + _logger.User("Import/update process complete!"); + + return success; + } + + /// + /// Import data into the database and return all files not found in the list + /// + /// List of files that were not found in the audit + private Dictionary, int> ImportData() + { + // Create the empty dictionary for file filtering and output + Dictionary, int> dbfiles = new Dictionary, int>(); + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) { dbc.Open(); + _logger.User("Populating reference objects"); - Dictionary hashes = new Dictionary(); - string query = "SELECT sha1, id FROM dats"; + // Populate the list of files in the database with Tuples (size, sha1, name) + string query = "SELECT id, size, sha1, name FROM dats"; using (SqliteCommand slc = new SqliteCommand(query, dbc)) { using (SqliteDataReader sldr = slc.ExecuteReader()) { while (sldr.Read()) { - hashes.Add(sldr.GetString(0), sldr.GetInt32(1)); + dbfiles.Add(Tuple.Create(sldr.GetInt64(1), sldr.GetString(2), sldr.GetString(3)), sldr.GetInt32(0)); } } } - SHA1 sha1 = SHA1.Create(); - query = "SELECT * FROM system"; + // Populate the list of systems + Dictionary systems = new Dictionary(); + query = "SELECT id, manufacturer, name FROM system"; using (SqliteCommand slc = new SqliteCommand(query, dbc)) { using (SqliteDataReader sldr = slc.ExecuteReader()) { while (sldr.Read()) { - int systemid = sldr.GetInt32(0); - string system = _datroot + Path.DirectorySeparatorChar + sldr.GetString(1) + " - " + sldr.GetString(2); - system = system.Trim(); + systems.Add(sldr.GetString(1) + " - " + sldr.GetString(2), sldr.GetInt32(0)); + } + } + } - _logger.User("System: " + system.Remove(0, 5)); + // Populate the list of sources (initial) + SortedDictionary sources = new SortedDictionary(); + sources.Add("default", 0); + query = "SELECT name, id FROM source"; + using (SqliteCommand sslc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader ssldr = sslc.ExecuteReader()) + { + while (ssldr.Read()) + { + sources.Add(ssldr.GetString(0).ToLowerInvariant(), ssldr.GetInt32(1)); + } + } + } - // Audit all DATs in the folder - foreach (string file in Directory.GetFiles(system, "*", SearchOption.AllDirectories)) + // Interate through each system and check files + SHA1 sha1 = SHA1.Create(); + foreach (KeyValuePair kv in systems) + { + _logger.User("Processing DATs for system: '" + kv.Key + "'"); + + // Set the folder to iterate through based on the DAT root + string folder = _datroot + Path.DirectorySeparatorChar + kv.Key.Trim(); + + // Audit all files in the folder + foreach (string file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories)) + { + // First get the file information for comparison + long size = (new FileInfo(file)).Length; + string hash = ""; + using (FileStream fs = File.Open(file, FileMode.Open)) + { + hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", ""); + } + + // If it's not in the list of known files, add it + if (!dbfiles.ContainsKey(Tuple.Create(size, hash, file))) + { + // First add the information to the database as is and return the new insert ID + _logger.Log("Adding file information for " + Path.GetFileName(file)); + + int hashid = -1; + query = @"INSERT INTO dats (size, sha1, name) +VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"')"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) { - string hash = ""; - using (FileStream fs = File.Open(file, FileMode.Open)) + slc.ExecuteNonQuery(); + } + + //query = "SELECT last_insertConstants.Rowid()"; + query = "SELECT last_insert_rowid()"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) { - hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", ""); - } - - // If the hash isn't in add it and all required information - int hashid = -1; - if (!hashes.ContainsKey(hash)) - { - _logger.Log("Adding file information for " + Path.GetFileName(file)); - - string squery = @"BEGIN; -INSERT INTO dats (size, sha1, name) -VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"'); -SELECT last_insertConstants.Rowid(); -COMMIT;"; - using (SqliteCommand sslc = new SqliteCommand(squery, dbc)) + if (sldr.Read()) { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - if (ssldr.Read()) - { - hashid = ssldr.GetInt32(0); - } - } - } - - // Add the hash to the temporary Dictionary - hashes.Add(hash, hashid); - - // Now try to determine the source for the file based on the name - string source = GetSourceFromFileName(Path.GetFileName(file)); - int sourceid = 0; - - SortedDictionary sources = new SortedDictionary(); - sources.Add("default", 0); - query = "SELECT name, id FROM source"; - using (SqliteCommand sslc = new SqliteCommand(query, dbc)) - { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - while (ssldr.Read()) - { - sources.Add(ssldr.GetString(0).ToLowerInvariant(), ssldr.GetInt32(1)); - } - } - } - - // Only if we're not ignoring new sources should be ask the user for input - if (!_ignore) - { - // We want to reset "Default" at this point, just in case - if (source.ToLowerInvariant() == "default") - { - source = ""; - } - - // If the source is blank, ask the user to supply one - while (source == "" && sourceid == 0) - { - Console.Clear(); - Build.Start("DATabaseTwo"); - - Console.WriteLine("Sources:"); - foreach (KeyValuePair pair in sources) - { - Console.WriteLine(" " + pair.Value + " - " + pair.Key); - } - Console.WriteLine("\nFor file name: " + Path.GetFileName(file)); - Console.Write("Select a source above or enter a new one: "); - source = Console.ReadLine(); - - Int32.TryParse(source, out sourceid); - - // If the value could be parsed, reset the source string - if (sourceid != 0) - { - source = ""; - } - - // If the source ID is set check to see if it's valid - if (sourceid != 0 && !sources.ContainsValue(sourceid)) - { - Console.WriteLine("Invalid selection: " + sourceid); - Console.ReadLine(); - sourceid = 0; - } - } - - // If the source isn't in, add it and get the insert id - if (source != "" && sourceid == 0 && !sources.ContainsKey(source.ToLowerInvariant())) - { - string tquery = @"BEGIN; -INSERT INTO source (name, url) -VALUES ('" + source + @"', ''); -SELECT last_insertConstants.Rowid(); -COMMIT;"; - using (SqliteCommand sslc = new SqliteCommand(tquery, dbc)) - { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - if (ssldr.Read()) - { - sourceid = ssldr.GetInt32(0); - } - } - } - - // Add the new source to the temporary Dictionary - sources.Add(source.ToLowerInvariant(), sourceid); - } - // Otherwise, get the ID - else if (source != "" && sourceid == 0 && sources.ContainsKey(source.ToLowerInvariant())) - { - try - { - sourceid = sources[source.ToLowerInvariant()]; - } - catch - { - sourceid = 0; - } - } - // Otherwise, we should already have an ID - } - else - { - try - { - sourceid = sources[source.ToLowerInvariant()]; - } - catch - { - sourceid = 0; - } - } - - // Add the source and system link to the database - string uquery = @"INSERT OR IGNORE INTO datsdata (id, key, value) -VALUES (" + hashid + ", 'source', '" + sourceid + @"'), -(" + hashid + ", 'system', '" + systemid + "')"; - using (SqliteCommand uslc = new SqliteCommand(uquery, dbc)) - { - uslc.ExecuteNonQuery(); + hashid = sldr.GetInt32(0); } } } + + // Next we try to figure out the source ID from the name + string possiblesource = GetSourceFromFileName(Path.GetFileName(file)); + + // Try to get the source ID from the name + int sourceid = (sources.ContainsKey(possiblesource.ToLowerInvariant()) ? sources[possiblesource] : 0); + + // If we have the "default" ID and we're not ignoring new sources, prompt for a source input + if (!_ignore) + { + // We want to reset "Default" at this point, just in case + if (possiblesource.ToLowerInvariant() == "default") + { + possiblesource = ""; + } + + // If the source is blank, ask the user to supply one + while (possiblesource == "" && sourceid == 0) + { + Console.Clear(); + Build.Start("DATabaseTwo"); + + Console.WriteLine("Sources:"); + foreach (KeyValuePair pair in sources) + { + Console.WriteLine(" " + pair.Value + " - " + Style.SentenceCase(pair.Key)); + } + Console.WriteLine("\nFor file name: " + Path.GetFileName(file)); + Console.Write("Select a source above or enter a new one: "); + possiblesource = Console.ReadLine(); + + Int32.TryParse(possiblesource, out sourceid); + + // If the value could be parsed, reset the source string + if (sourceid != 0) + { + possiblesource = ""; + } + + // If the source ID is set check to see if it's valid + if (sourceid != 0 && !sources.ContainsValue(sourceid)) + { + Console.WriteLine("Invalid selection: " + sourceid); + Console.ReadLine(); + sourceid = 0; + } + } + + // If we have a non-empty possible source and it's in the database, get the id + if (possiblesource != "" && sources.ContainsKey(possiblesource.ToLowerInvariant())) + { + sourceid = sources[possiblesource.ToLowerInvariant()]; + } + + // If we have a non-empty possible source and it's not in the database, insert and get the id + else if (possiblesource != "" && !sources.ContainsKey(possiblesource.ToLowerInvariant())) + { + query = @"BEGIN; +INSERT INTO source (name, url) +VALUES ('" + possiblesource + @"', ''); +SELECT last_insertConstants.Rowid(); +COMMIT;"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + if (sldr.Read()) + { + sourceid = sldr.GetInt32(0); + } + } + } + + // Add the new source to the current dictionary + sources.Add(possiblesource.ToLowerInvariant(), sourceid); + } + } + + // Now that we have a source ID, we can add the mappings for system and source to the database + query = @"INSERT OR IGNORE INTO datsdata (id, key, value) +VALUES (" + hashid + ", 'source', '" + sourceid + @"'), +(" + hashid + ", 'system', '" + kv.Value + "')"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + slc.ExecuteNonQuery(); + } + } + + // Otherwise, remove it from the list of found items + else + { + dbfiles.Remove(Tuple.Create(size, hash, file)); } } } } - _logger.User("Import/update process complete!"); + return dbfiles; + } - return true; + /// + /// Remove all data associated with various files + /// + /// List of file identifiers to remove from the database + /// True if everything went well, false otherwise + private bool RemoveData(Dictionary, int> missing) + { + bool success = true; + + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) + { + dbc.Open(); + + // Get a comma-separated list of IDs from the input files + string idlist = String.Join(",", missing.Values); + + // Now remove all of the files from the database + string query = @"BEGIN; +DELETE FROM datsdata WHERE id IN (" + idlist + @"); +DELETE FROM dats WHERE id IN (" + idlist + @"); +COMMIT;"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + slc.ExecuteNonQuery(); + } + } + + return success; } /// @@ -276,6 +325,12 @@ VALUES (" + hashid + ", 'source', '" + sourceid + @"'), else if (Regex.IsMatch(filename, Constants.GoodXmlPattern)) { fileinfo = Regex.Match(filename, Constants.GoodXmlPattern).Groups; + if (!Remapping.Good.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped."); + return source; + } + source = "Good"; } else if (Regex.IsMatch(filename, Constants.MaybeIntroPattern)) { diff --git a/README.MD b/README.MD index 56309dd7..ca2a90e2 100644 --- a/README.MD +++ b/README.MD @@ -19,7 +19,7 @@ A bare-bones attempt at providing a true GUI experience for the SabreTools suite

The main tool of the SabreTools suite. Performs the majority of the core features of the parent project, including the following:

    -
  • Importing and Generating DAT files in ClrMamePro and XML formats
    • +
  • Importing and Generating DAT files in ClrMamePro and XML formats (current version formerly DATabaseTwo)
  • Converting DATs from any format to ClrMamePro, Logiqx XML, SabreDAT XML, and to missfile (last part formerly DatToMiss; requested by Obiwantje)
  • Trim DAT entries and optionally merge into a single game (formerly SingleGame; requested by Kludge)
  • Split a DAT using two different file extensions within the DAT (formerly DatSplit)
    • @@ -39,10 +39,6 @@ The main tool of the SabreTools suite. Performs the majority of the core feature
This tool can both be used with command line parameters or with the built-in text menu system. -DATabaseTwo -

-An in-progress replacement for the Import and Generate functions of DATabase. This is the recommended version to use for mass DAT management but it is still lacking some advanced features found only in the DATabase version. - Headerer

A simple auxilary tool that detects and removes headers and also restores headers for the following systems: diff --git a/SabreHelper/DBTools.cs b/SabreHelper/DBTools.cs index 20106e0e..44e678af 100644 --- a/SabreHelper/DBTools.cs +++ b/SabreHelper/DBTools.cs @@ -133,7 +133,7 @@ CREATE TABLE IF NOT EXISTS system ( { return sslc.ExecuteNonQuery() >= 1; } - } + } } } } diff --git a/SabreHelper/Data/Constants.cs b/SabreHelper/Data/Constants.cs index 0db5a6da..aa7dfdc1 100644 --- a/SabreHelper/Data/Constants.cs +++ b/SabreHelper/Data/Constants.cs @@ -20,16 +20,16 @@ public static string GoodXmlPattern = @"^(Good.*?)_.*\.xml"; public static string MamePattern = @"^(.*)\.xml$"; public static string MaybeIntroPattern = @"(.*?) \[T-En\].*\((\d{8})\)\.dat$"; - public static string NoIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\)\.dat$"; - public static string NoIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).dat"; - public static string NoIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\)\.dat"; + public static string NoIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\).*\.dat$"; + public static string NoIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).*.dat"; + public static string NoIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\).*\.dat"; public static string NonGoodPattern = @"^(NonGood.*?)( .*?)?.xml"; public static string NonGoodSpecialPattern = @"^(NonGood.*?)( .*)?.dat"; public static string RedumpPattern = @"^(.*?) \((\d{8} \d{2}-\d{2}-\d{2})\)\.dat$"; public static string RedumpBiosPattern = @"^(.*?) \(\d+\) \((\d{4}-\d{2}-\d{2})\)\.dat$"; - public static string TosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; - public static string TosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; - public static string TosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; + public static string TosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$"; + public static string TosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$"; + public static string TosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\).*\.dat$"; public static string TruripPattern = @"^(.*) - .* \(trurip_XML\)\.dat$"; public static string ZandroPattern = @"^SMW-.*.xml"; diff --git a/SabreHelper/Interfaces/IImport.cs b/SabreHelper/Interfaces/IImport.cs index ca6ce6ca..5aa477e9 100644 --- a/SabreHelper/Interfaces/IImport.cs +++ b/SabreHelper/Interfaces/IImport.cs @@ -2,6 +2,6 @@ { public interface IImport { - bool ImportData(); + bool UpdateDatabase(); } } diff --git a/SabreHelper/Mappings/NoIntro.xml b/SabreHelper/Mappings/NoIntro.xml index f8bd303a..a0a7a919 100644 --- a/SabreHelper/Mappings/NoIntro.xml +++ b/SabreHelper/Mappings/NoIntro.xml @@ -38,6 +38,7 @@ + diff --git a/SabreHelper/Style.cs b/SabreHelper/Style.cs index 65e46771..1b6cfa39 100644 --- a/SabreHelper/Style.cs +++ b/SabreHelper/Style.cs @@ -218,5 +218,22 @@ namespace SabreTools.Helper // Return formatted number with suffix return readable.ToString("0.### ") + suffix; } + + ///

+ /// Converts a string to sentence case. + /// + /// The string to convert. + /// A string representing a sentence case string + /// http://stackoverflow.com/questions/3141426/net-method-to-convert-a-string-to-sentence-case + public static string SentenceCase(string input) + { + if (input.Length < 1) + { + return input; + } + + string sentence = input.ToLower(); + return sentence[0].ToString().ToUpper() + sentence.Substring(1); + } } } diff --git a/SabreHelper/dats.sqlite b/SabreHelper/dats.sqlite index 659108b0..b49f751d 100644 Binary files a/SabreHelper/dats.sqlite and b/SabreHelper/dats.sqlite differ