diff --git a/DATabaseTwo/DATabaseTwo.cs b/DATabaseTwo/DATabaseTwo.cs index 78f5be6c..920577e5 100644 --- a/DATabaseTwo/DATabaseTwo.cs +++ b/DATabaseTwo/DATabaseTwo.cs @@ -2,9 +2,6 @@ using System.Collections.Generic; using Mono.Data.Sqlite; using System.IO; -using System.Linq; -using System.Security.Cryptography; -using System.Text.RegularExpressions; using SabreTools.Helper; @@ -19,26 +16,6 @@ namespace SabreTools private static string _connectionString = "Data Source=" + _dbName + ";Version = 3;"; private static Logger _logger; - // Regex File Name Patterns - private static string _defaultPattern = @"^(.+?) - (.+?) \((.*) (.*)\)\.dat$"; - private static string _defaultSpecialPattern = @"^(.+?) - (.+?) \((.*) (.*)\)\.xml$"; - private static string _goodPattern = @"^(Good.*?)_.*\.dat"; - private static string _goodXmlPattern = @"^(Good.*?)_.*\.xml"; - private static string _mamePattern = @"^(.*)\.xml$"; - private static string _maybeIntroPattern = @"(.*?) \[T-En\].*\((\d{8})\)\.dat$"; - private static string _noIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\)\.dat$"; - private static string _noIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).dat"; - private static string _noIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\)\.dat"; - private static string _nonGoodPattern = @"^(NonGood.*?)( .*?)?.xml"; - private static string _nonGoodSpecialPattern = @"^(NonGood.*?)( .*)?.dat"; - private static string _redumpPattern = @"^(.*?) \((\d{8} \d{2}-\d{2}-\d{2})\)\.dat$"; - private static string _redumpBiosPattern = @"^(.*?) \(\d+\) \((\d{4}-\d{2}-\d{2})\)\.dat$"; - private static string _tosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; - private static string _tosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; - private static string _tosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; - private static string _truripPattern = @"^(.*) - .* \(trurip_XML\)\.dat$"; - private static string _zandroPattern = @"^SMW-.*.xml"; - public static void Main(string[] args) { Console.Clear(); @@ -143,18 +120,17 @@ namespace SabreTools // If we want to generate all DATs else if (genall) { - Import(ignore); + InitImport(ignore); InitGenerateAll(norename, old); } // If we want to generate a DAT else if (gen) { - Import(ignore); + InitImport(ignore); InitGenerate(system, norename, old); } - _logger.Close(); } @@ -247,7 +223,7 @@ Make a selection: break; case "2": Console.Clear(); - Import(ignore); + InitImport(ignore); Console.Write("\nPress any key to continue..."); Console.ReadKey(); ignore = false; @@ -320,6 +296,16 @@ Make a selection: #region Function Methods + /// + /// Wrap importing and updating DATs + /// + /// + private static void InitImport(bool ignore) + { + Import imp = new Import(_datroot, _connectionString, _logger, ignore); + imp.ImportData(); + } + /// /// Wrap generating a DAT from the library /// @@ -328,171 +314,8 @@ Make a selection: /// True if the output file should be in ClrMamePro format (default false) private static void InitGenerate(string systemid, bool norename, bool old) { - string name = ""; - string path = _datroot; - - // If the System ID isn't set, then we will merge everything - if (systemid != "") - { - string system = ""; - - // First get the name of the system, if possible - string query = "SELECT manufacturer, name FROM system WHERE id=" + systemid; - using (SqliteConnection dbc = new SqliteConnection(_connectionString)) - { - dbc.Open(); - - using (SqliteCommand slc = new SqliteCommand(query, dbc)) - { - using (SqliteDataReader sldr = slc.ExecuteReader()) - { - if (sldr.Read()) - { - system = sldr.GetString(0) + " - " + sldr.GetString(1); - } - } - } - } - - // If we didn't find anything, then return - if (system == "") - { - _logger.Warning("No system could be found with id " + systemid); - return; - } - - path += Path.DirectorySeparatorChar + system.Trim(); - name = system.Trim(); - } - else - { - name = "ALL"; - } - - // Get the rest of the info as well - string date = DateTime.Now.ToString("yyyyMMddHHmmss"); - string description = name + " (merged " + date + ")"; - name += " (merged)"; - - // For good measure, get all sources - Dictionary sources = new Dictionary(); - sources.Add(0, "Default"); - - string squery = "SELECT id, name FROM source"; - using (SqliteConnection dbc = new SqliteConnection(_connectionString)) - { - dbc.Open(); - - using (SqliteCommand slc = new SqliteCommand(squery, dbc)) - { - using (SqliteDataReader sldr = slc.ExecuteReader()) - { - while (sldr.Read()) - { - sources.Add(sldr.GetInt32(0), sldr.GetString(1)); - } - } - } - } - - // Get a list of files to sourceid mappings - Dictionary sourcemap = new Dictionary(); - using (SqliteConnection dbc = new SqliteConnection(_connectionString)) - { - dbc.Open(); - - string tquery = "SELECT DISTINCT dats.name, datsdata.value FROM dats JOIN datsdata ON dats.id=datsdata.id WHERE key='source'"; - using (SqliteCommand slc = new SqliteCommand(tquery, dbc)) - { - using (SqliteDataReader sldr = slc.ExecuteReader()) - { - while (sldr.Read()) - { - string tempname = sldr.GetString(0); - string tempval = sldr.GetString(1); - if (!sourcemap.ContainsKey(tempname)) - { - sourcemap.Add(tempname, tempval); - } - } - } - } - } - - // Now read in all of the files - Dictionary> roms = new Dictionary>(); - foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories)) - { - int tempSrcId = 0; - if (sourcemap.ContainsKey(file)) - { - Int32.TryParse(sourcemap[file], out tempSrcId); - } - roms = RomManipulation.ParseDict(file, 0, tempSrcId, roms, _logger); - } - - // If the dictionary is empty for any reason, tell the user and exit - if (roms.Keys.Count == 0 || roms.Count == 0) - { - _logger.Log("No roms found for system ID " + systemid); - return; - } - - // Now process all of the roms - _logger.Log("Cleaning rom data"); - List keys = roms.Keys.ToList(); - foreach (string key in keys) - { - List temp = new List(); - List newroms = roms[key]; - for (int i = 0; i < newroms.Count; i++) - { - RomData rom = newroms[i]; - - // In the case that the RomData is incomplete, skip it - if (rom.Name == null || rom.Game == null) - { - continue; - } - - // WOD origninally stripped out any subdirs from the imported files, we do the same - rom.Name = Path.GetFileName(rom.Name); - - // Run the name through the filters to make sure that it's correct - rom.Name = Style.NormalizeChars(rom.Name); - rom.Name = Style.RussianToLatin(rom.Name); - rom.Name = Regex.Replace(rom.Name, @"(.*) \.(.*)", "$1.$2"); - - // WoD gets rid of anything past the first "(" or "[" as the name, we will do the same - string stripPattern = @"(([[(].*[\)\]] )?([^([]+))"; - Regex stripRegex = new Regex(stripPattern); - Match stripMatch = stripRegex.Match(rom.Game); - rom.Game = stripMatch.Groups[1].Value; - - // Run the name through the filters to make sure that it's correct - rom.Game = Style.NormalizeChars(rom.Game); - rom.Game = Style.RussianToLatin(rom.Game); - rom.Game = Style.SearchPattern(rom.Game); - rom.Game = rom.Game.TrimEnd().TrimStart(); - - if (!norename) - { - rom.Game += " [" + sources[rom.SourceID] + "]"; - } - - // If a game has source "0" it's Default. Make this Int32.MaxValue for sorting purposes - if (rom.SourceID == 0) - { - rom.SourceID = Int32.MaxValue; - } - - temp.Add(rom); - } - roms[key] = temp; - } - - // Then write out the file - Output.WriteToDatFromDict(name, description, "", date, "SabreTools", "SabreTools", false, old, true, _outroot, roms, _logger, norename); + Generate gen = new Generate(systemid, "" /* sourceid */, _datroot, _outroot, _connectionString, _logger, norename, old); + gen.Export(); } /// @@ -614,348 +437,6 @@ ORDER BY system.manufacturer, system.name"; } } - /// - /// Perform initial or incremental import of DATs in the root folder - /// - /// False if each DAT that has no defined source asks for user input (default), true otherwise - private static void Import(bool ignore = false) - { - _logger.Log("Beginning import/update process"); - using (SqliteConnection dbc = new SqliteConnection(_connectionString)) - { - dbc.Open(); - - Dictionary hashes = new Dictionary(); - string query = "SELECT sha1, id FROM dats"; - using (SqliteCommand slc = new SqliteCommand(query, dbc)) - { - using (SqliteDataReader sldr = slc.ExecuteReader()) - { - while (sldr.Read()) - { - hashes.Add(sldr.GetString(0), sldr.GetInt32(1)); - } - } - } - - SHA1 sha1 = SHA1.Create(); - query = "SELECT * FROM system"; - using (SqliteCommand slc = new SqliteCommand(query, dbc)) - { - using (SqliteDataReader sldr = slc.ExecuteReader()) - { - while (sldr.Read()) - { - int systemid = sldr.GetInt32(0); - string system = _datroot + Path.DirectorySeparatorChar + sldr.GetString(1) + " - " + sldr.GetString(2); - system = system.Trim(); - - _logger.Log("System: " + system.Remove(0, 5)); - - // Audit all DATs in the folder - foreach (string file in Directory.GetFiles(system, "*", SearchOption.AllDirectories)) - { - string hash = ""; - using (FileStream fs = File.Open(file, FileMode.Open)) - { - hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", ""); - } - - // If the hash isn't in add it and all required information - int hashid = -1; - if (!hashes.ContainsKey(hash)) - { - _logger.Log("Adding file information for " + Path.GetFileName(file)); - - string squery = @"BEGIN; -INSERT INTO dats (size, sha1, name) -VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"'); -SELECT last_insert_rowid(); -COMMIT;"; - using (SqliteCommand sslc = new SqliteCommand(squery, dbc)) - { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - if (ssldr.Read()) - { - hashid = ssldr.GetInt32(0); - } - } - } - - // Add the hash to the temporary Dictionary - hashes.Add(hash, hashid); - - // If we don't care about source, stop here - if (ignore) - { - continue; - } - - // Now try to determine the source for the file based on the name - string source = GetSourceFromFileName(Path.GetFileName(file)); - int sourceid = 0; - - Dictionary sources = new Dictionary(); - query = "SELECT name, id FROM source"; - using (SqliteCommand sslc = new SqliteCommand(query, dbc)) - { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - while (ssldr.Read()) - { - sources.Add(ssldr.GetString(0), ssldr.GetInt32(1)); - } - } - } - - // If the source is blank, ask the user to supply one - while (source == "" && sourceid == 0) - { - Console.Clear(); - Build.Start("DATabaseTwo"); - - Console.WriteLine("Sources:"); - foreach (KeyValuePair pair in sources) - { - Console.WriteLine(" " + pair.Value + " - " + pair.Key); - } - Console.WriteLine("\nFor file name: " + Path.GetFileName(file)); - Console.Write("Select a source above or enter a new one: "); - source = Console.ReadLine(); - - Int32.TryParse(source, out sourceid); - - // If the value could be parsed, reset the source string - if (sourceid != 0) - { - source = ""; - } - - // If the source ID is set check to see if it's valid - if (sourceid != 0 && !sources.ContainsValue(sourceid)) - { - Console.WriteLine("Invalid selection: " + sourceid); - Console.ReadLine(); - sourceid = 0; - } - } - - // If the source isn't in, add it and get the insert id - if (source != "" && sourceid == 0 && !sources.ContainsKey(source)) - { - string tquery = @"BEGIN; -INSERT INTO source (name, url) -VALUES ('" + source + @"', ''); -SELECT last_insert_rowid(); -COMMIT;"; - using (SqliteCommand sslc = new SqliteCommand(tquery, dbc)) - { - using (SqliteDataReader ssldr = sslc.ExecuteReader()) - { - if (ssldr.Read()) - { - sourceid = ssldr.GetInt32(0); - } - } - } - - // Add the new source to the temporary Dictionary - sources.Add(source, sourceid); - } - // Otherwise, get the ID - else if (source != "" && sourceid == 0 && sources.ContainsKey(source)) - { - sourceid = sources[source]; - } - // Otherwise, we should already have an ID - - // Add the source and system link to the database - string uquery = @"INSERT OR IGNORE INTO datsdata (id, key, value) -VALUES (" + hashid + ", 'source', '" + sourceid + @"'), -(" + hashid + ", 'system', '" + systemid + "')"; - using (SqliteCommand uslc = new SqliteCommand(uquery, dbc)) - { - uslc.ExecuteNonQuery(); - } - } - } - } - } - } - } - - _logger.Log("Import/update process complete!"); - } - - /// - /// Determine the source name from the file name, if possible - /// - /// The name of the file to be checked - /// The name of the source if determined, blank otherwise - private static string GetSourceFromFileName (string filename) - { - string source = ""; - - // Determine which dattype we have - GroupCollection fileinfo; - - if (Regex.IsMatch(filename, _nonGoodPattern)) - { - fileinfo = Regex.Match(filename, _nonGoodPattern).Groups; - if (!Remapping.NonGood.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped."); - return source; - } - source = "NonGood"; - } - else if (Regex.IsMatch(filename, _nonGoodSpecialPattern)) - { - fileinfo = Regex.Match(filename, _nonGoodSpecialPattern).Groups; - if (!Remapping.NonGood.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped."); - return source; - } - source = "NonGood"; - } - else if (Regex.IsMatch(filename, _goodPattern)) - { - fileinfo = Regex.Match(filename, _goodPattern).Groups; - if (!Remapping.Good.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped."); - return source; - } - source = "Good"; - } - else if (Regex.IsMatch(filename, _goodXmlPattern)) - { - fileinfo = Regex.Match(filename, _goodXmlPattern).Groups; - } - else if (Regex.IsMatch(filename, _maybeIntroPattern)) - { - fileinfo = Regex.Match(filename, _maybeIntroPattern).Groups; - if (!Remapping.MaybeIntro.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Maybe-Intro but could not be mapped."); - return source; - } - source = "Maybe-Intro"; - } - else if (Regex.IsMatch(filename, _noIntroPattern)) - { - fileinfo = Regex.Match(filename, _noIntroPattern).Groups; - if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); - return source; - } - source = "no-Intro"; - } - // For numbered DATs only - else if (Regex.IsMatch(filename, _noIntroNumberedPattern)) - { - fileinfo = Regex.Match(filename, _noIntroNumberedPattern).Groups; - if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); - return source; - } - source = "no-Intro"; - } - // For N-Gage and Gizmondo only - else if (Regex.IsMatch(filename, _noIntroSpecialPattern)) - { - fileinfo = Regex.Match(filename, _noIntroSpecialPattern).Groups; - if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); - return source; - } - source = "no-Intro"; - } - else if (Regex.IsMatch(filename, _redumpPattern)) - { - fileinfo = Regex.Match(filename, _redumpPattern).Groups; - if (!Remapping.Redump.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped."); - return source; - } - source = "Redump"; - } - // For special BIOSes only - else if (Regex.IsMatch(filename, _redumpBiosPattern)) - { - fileinfo = Regex.Match(filename, _redumpBiosPattern).Groups; - if (!Remapping.Redump.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped."); - return source; - } - source = "Redump"; - } - else if (Regex.IsMatch(filename, _tosecPattern)) - { - fileinfo = Regex.Match(filename, _tosecPattern).Groups; - if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) - { - // Handle special case mappings found only in TOSEC - fileinfo = Regex.Match(filename, _tosecSpecialPatternA).Groups; - - if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) - { - fileinfo = Regex.Match(filename, _tosecSpecialPatternB).Groups; - - if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as TOSEC but could not be mapped."); - return source; - } - } - } - source = "TOSEC"; - } - else if (Regex.IsMatch(filename, _truripPattern)) - { - fileinfo = Regex.Match(filename, _truripPattern).Groups; - if (!Remapping.TruRip.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as TruRip but could not be mapped."); - return source; - } - source = "trurip"; - } - else if (Regex.IsMatch(filename, _zandroPattern)) - { - source = "Zandro"; - } - else if (Regex.IsMatch(filename, _defaultPattern)) - { - fileinfo = Regex.Match(filename, _defaultPattern).Groups; - source = fileinfo[3].Value; - } - else if (Regex.IsMatch(filename, _defaultSpecialPattern)) - { - fileinfo = Regex.Match(filename, _defaultSpecialPattern).Groups; - source = fileinfo[3].Value; - } - else if (Regex.IsMatch(filename, _mamePattern)) - { - fileinfo = Regex.Match(filename, _mamePattern).Groups; - if (!Remapping.MAME.ContainsKey(fileinfo[1].Value)) - { - _logger.Warning("The filename " + fileinfo[1].Value + " was matched as MAME but could not be mapped."); - return source; - } - source = "MAME"; - } - - return source; - } - #endregion } } diff --git a/DATabaseTwo/DATabaseTwo.csproj b/DATabaseTwo/DATabaseTwo.csproj index e8c572e2..72db8076 100644 --- a/DATabaseTwo/DATabaseTwo.csproj +++ b/DATabaseTwo/DATabaseTwo.csproj @@ -77,6 +77,8 @@ + + diff --git a/DATabaseTwo/Generate.cs b/DATabaseTwo/Generate.cs new file mode 100644 index 00000000..79d8fcbf --- /dev/null +++ b/DATabaseTwo/Generate.cs @@ -0,0 +1,225 @@ +using System; +using System.Collections.Generic; +using Mono.Data.Sqlite; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using SabreTools.Helper; + +namespace SabreTools +{ + public class Generate + { + // Private instance variables + private string _systemid; + private string _sourceid; + private string _datroot; + private string _outroot; + private string _connectionString; + private bool _norename; + private bool _old; + + // Private required variables + private Logger _logger; + + /// + /// Initialize a Generate object with the given information + /// + /// String representing the system id (blank means all) + /// String representing the source id (blank means all) [CURRENTLY UNUSED] + /// Root directory where all DAT files are held + /// Root directory where new DAT files are output + /// Connection string for SQLite + /// Logger object for file or console output + /// True if files should not be renamed with system and/or source in merged mode (default false) + /// True if the output file should be in ClrMamePro format (default false) + public Generate(string systemid, string sourceid, string datroot, string outroot, string connectionString, Logger logger, bool norename = false, bool old = false) + { + _systemid = systemid; + _sourceid = sourceid; + _datroot = datroot; + _outroot = outroot; + _connectionString = connectionString; + _logger = logger; + _norename = norename; + _old = old; + } + + /// + /// Generate a DAT file that is represented by the data in the Generate object. + /// + /// True if the file could be created, false otherwise + public bool Export() + { + string name = ""; + string path = _datroot; + + // If the System ID isn't set, then we will merge everything + if (_systemid != "") + { + string system = ""; + + // First get the name of the system, if possible + string query = "SELECT manufacturer, name FROM system WHERE id=" + _systemid; + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) + { + dbc.Open(); + + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + if (sldr.Read()) + { + system = sldr.GetString(0) + " - " + sldr.GetString(1); + } + } + } + } + + // If we didn't find anything, then return + if (system == "") + { + _logger.Warning("No system could be found with id " + _systemid); + return false; + } + + path += Path.DirectorySeparatorChar + system.Trim(); + name = system.Trim(); + } + else + { + name = "ALL"; + } + + // Get the rest of the info as well + string date = DateTime.Now.ToString("yyyyMMddHHmmss"); + string description = name + " (merged " + date + ")"; + name += " (merged)"; + + // For good measure, get all sources + Dictionary sources = new Dictionary(); + sources.Add(0, "Default"); + + string squery = "SELECT id, name FROM source"; + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) + { + dbc.Open(); + + using (SqliteCommand slc = new SqliteCommand(squery, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + while (sldr.Read()) + { + sources.Add(sldr.GetInt32(0), sldr.GetString(1)); + } + } + } + } + + // Get a list of files to sourceid mappings + Dictionary sourcemap = new Dictionary(); + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) + { + dbc.Open(); + + string tquery = "SELECT DISTINCT dats.name, datsdata.value FROM dats JOIN datsdata ON dats.id=datsdata.id WHERE key='source'"; + using (SqliteCommand slc = new SqliteCommand(tquery, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + while (sldr.Read()) + { + string tempname = sldr.GetString(0); + string tempval = sldr.GetString(1); + if (!sourcemap.ContainsKey(tempname)) + { + sourcemap.Add(tempname, tempval); + } + } + } + } + } + + // Now read in all of the files + Dictionary> roms = new Dictionary>(); + foreach (string file in Directory.GetFiles(path, "*", SearchOption.AllDirectories)) + { + int tempSrcId = 0; + if (sourcemap.ContainsKey(file)) + { + Int32.TryParse(sourcemap[file], out tempSrcId); + } + roms = RomManipulation.ParseDict(file, 0, tempSrcId, roms, _logger); + } + + // If the dictionary is empty for any reason, tell the user and exit + if (roms.Keys.Count == 0 || roms.Count == 0) + { + _logger.Log("No roms found for system ID " + _systemid); + return false; + } + + // Now process all of the roms + _logger.Log("Cleaning rom data"); + List keys = roms.Keys.ToList(); + foreach (string key in keys) + { + List temp = new List(); + List newroms = roms[key]; + for (int i = 0; i < newroms.Count; i++) + { + RomData rom = newroms[i]; + + // In the case that the RomData is incomplete, skip it + if (rom.Name == null || rom.Game == null) + { + continue; + } + + // WOD origninally stripped out any subdirs from the imported files, we do the same + rom.Name = Path.GetFileName(rom.Name); + + // Run the name through the filters to make sure that it's correct + rom.Name = Style.NormalizeChars(rom.Name); + rom.Name = Style.RussianToLatin(rom.Name); + rom.Name = Regex.Replace(rom.Name, @"(.*) \.(.*)", "$1.$2"); + + // WoD gets rid of anything past the first "(" or "[" as the name, we will do the same + string stripPattern = @"(([[(].*[\)\]] )?([^([]+))"; + Regex stripRegex = new Regex(stripPattern); + Match stripMatch = stripRegex.Match(rom.Game); + rom.Game = stripMatch.Groups[1].Value; + + // Run the name through the filters to make sure that it's correct + rom.Game = Style.NormalizeChars(rom.Game); + rom.Game = Style.RussianToLatin(rom.Game); + rom.Game = Style.SearchPattern(rom.Game); + rom.Game = rom.Game.TrimEnd().TrimStart(); + + if (!_norename) + { + rom.Game += " [" + sources[rom.SourceID] + "]"; + } + + // If a game has source "0" it's Default. Make this Int32.MaxValue for sorting purposes + if (rom.SourceID == 0) + { + rom.SourceID = Int32.MaxValue; + } + + temp.Add(rom); + } + roms[key] = temp; + } + + // Then write out the file + Output.WriteToDatFromDict(name, description, "", date, "SabreTools", "SabreTools", false, _old, true, _outroot, roms, _logger, _norename); + + return true; + } + } +} diff --git a/DATabaseTwo/Import.cs b/DATabaseTwo/Import.cs new file mode 100644 index 00000000..c65fe531 --- /dev/null +++ b/DATabaseTwo/Import.cs @@ -0,0 +1,394 @@ +using System; +using System.Collections.Generic; +using Mono.Data.Sqlite; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using SabreTools.Helper; + +namespace SabreTools +{ + public class Import + { + // Private instance variables + private string _datroot; + private string _connectionString; + private Logger _logger; + private bool _ignore; + + // Regex File Name Patterns + private static string _defaultPattern = @"^(.+?) - (.+?) \((.*) (.*)\)\.dat$"; + private static string _defaultSpecialPattern = @"^(.+?) - (.+?) \((.*) (.*)\)\.xml$"; + private static string _goodPattern = @"^(Good.*?)_.*\.dat"; + private static string _goodXmlPattern = @"^(Good.*?)_.*\.xml"; + private static string _mamePattern = @"^(.*)\.xml$"; + private static string _maybeIntroPattern = @"(.*?) \[T-En\].*\((\d{8})\)\.dat$"; + private static string _noIntroPattern = @"^(.*?) \((\d{8}-\d{6})_CM\)\.dat$"; + private static string _noIntroNumberedPattern = @"(.*? - .*?) \(\d.*?_CM\).dat"; + private static string _noIntroSpecialPattern = @"(.*? - .*?) \((\d{8})\)\.dat"; + private static string _nonGoodPattern = @"^(NonGood.*?)( .*?)?.xml"; + private static string _nonGoodSpecialPattern = @"^(NonGood.*?)( .*)?.dat"; + private static string _redumpPattern = @"^(.*?) \((\d{8} \d{2}-\d{2}-\d{2})\)\.dat$"; + private static string _redumpBiosPattern = @"^(.*?) \(\d+\) \((\d{4}-\d{2}-\d{2})\)\.dat$"; + private static string _tosecPattern = @"^(.*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; + private static string _tosecSpecialPatternA = @"^(.*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; + private static string _tosecSpecialPatternB = @"^(.*? - .*? - .*?) - .* \(TOSEC-v(\d{4}-\d{2}-\d{2})_CM\)\.dat$"; + private static string _truripPattern = @"^(.*) - .* \(trurip_XML\)\.dat$"; + private static string _zandroPattern = @"^SMW-.*.xml"; + + public Import(string datroot, string connectionString, Logger logger, bool ignore = false) + { + _datroot = datroot; + _connectionString = connectionString; + _logger = logger; + _ignore = ignore; + } + + /// + /// Perform initial or incremental import of DATs in the root folder + /// + /// False if each DAT that has no defined source asks for user input (default), true otherwise + public bool ImportData(bool ignore = false) + { + _logger.Log("Beginning import/update process"); + using (SqliteConnection dbc = new SqliteConnection(_connectionString)) + { + dbc.Open(); + + Dictionary hashes = new Dictionary(); + string query = "SELECT sha1, id FROM dats"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + while (sldr.Read()) + { + hashes.Add(sldr.GetString(0), sldr.GetInt32(1)); + } + } + } + + SHA1 sha1 = SHA1.Create(); + query = "SELECT * FROM system"; + using (SqliteCommand slc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader sldr = slc.ExecuteReader()) + { + while (sldr.Read()) + { + int systemid = sldr.GetInt32(0); + string system = _datroot + Path.DirectorySeparatorChar + sldr.GetString(1) + " - " + sldr.GetString(2); + system = system.Trim(); + + _logger.Log("System: " + system.Remove(0, 5)); + + // Audit all DATs in the folder + foreach (string file in Directory.GetFiles(system, "*", SearchOption.AllDirectories)) + { + string hash = ""; + using (FileStream fs = File.Open(file, FileMode.Open)) + { + hash = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", ""); + } + + // If the hash isn't in add it and all required information + int hashid = -1; + if (!hashes.ContainsKey(hash)) + { + _logger.Log("Adding file information for " + Path.GetFileName(file)); + + string squery = @"BEGIN; +INSERT INTO dats (size, sha1, name) +VALUES (" + (new FileInfo(file)).Length + ", '" + hash + "', '" + file.Replace("'", "''") + @"'); +SELECT last_insert_rowid(); +COMMIT;"; + using (SqliteCommand sslc = new SqliteCommand(squery, dbc)) + { + using (SqliteDataReader ssldr = sslc.ExecuteReader()) + { + if (ssldr.Read()) + { + hashid = ssldr.GetInt32(0); + } + } + } + + // Add the hash to the temporary Dictionary + hashes.Add(hash, hashid); + + // If we don't care about source, stop here + if (ignore) + { + continue; + } + + // Now try to determine the source for the file based on the name + string source = GetSourceFromFileName(Path.GetFileName(file)); + int sourceid = 0; + + Dictionary sources = new Dictionary(); + query = "SELECT name, id FROM source"; + using (SqliteCommand sslc = new SqliteCommand(query, dbc)) + { + using (SqliteDataReader ssldr = sslc.ExecuteReader()) + { + while (ssldr.Read()) + { + sources.Add(ssldr.GetString(0), ssldr.GetInt32(1)); + } + } + } + + // If the source is blank, ask the user to supply one + while (source == "" && sourceid == 0) + { + Console.Clear(); + Build.Start("DATabaseTwo"); + + Console.WriteLine("Sources:"); + foreach (KeyValuePair pair in sources) + { + Console.WriteLine(" " + pair.Value + " - " + pair.Key); + } + Console.WriteLine("\nFor file name: " + Path.GetFileName(file)); + Console.Write("Select a source above or enter a new one: "); + source = Console.ReadLine(); + + Int32.TryParse(source, out sourceid); + + // If the value could be parsed, reset the source string + if (sourceid != 0) + { + source = ""; + } + + // If the source ID is set check to see if it's valid + if (sourceid != 0 && !sources.ContainsValue(sourceid)) + { + Console.WriteLine("Invalid selection: " + sourceid); + Console.ReadLine(); + sourceid = 0; + } + } + + // If the source isn't in, add it and get the insert id + if (source != "" && sourceid == 0 && !sources.ContainsKey(source)) + { + string tquery = @"BEGIN; +INSERT INTO source (name, url) +VALUES ('" + source + @"', ''); +SELECT last_insert_rowid(); +COMMIT;"; + using (SqliteCommand sslc = new SqliteCommand(tquery, dbc)) + { + using (SqliteDataReader ssldr = sslc.ExecuteReader()) + { + if (ssldr.Read()) + { + sourceid = ssldr.GetInt32(0); + } + } + } + + // Add the new source to the temporary Dictionary + sources.Add(source, sourceid); + } + // Otherwise, get the ID + else if (source != "" && sourceid == 0 && sources.ContainsKey(source)) + { + sourceid = sources[source]; + } + // Otherwise, we should already have an ID + + // Add the source and system link to the database + string uquery = @"INSERT OR IGNORE INTO datsdata (id, key, value) +VALUES (" + hashid + ", 'source', '" + sourceid + @"'), +(" + hashid + ", 'system', '" + systemid + "')"; + using (SqliteCommand uslc = new SqliteCommand(uquery, dbc)) + { + uslc.ExecuteNonQuery(); + } + } + } + } + } + } + } + + _logger.Log("Import/update process complete!"); + + return true; + } + + /// + /// Determine the source name from the file name, if possible + /// + /// The name of the file to be checked + /// The name of the source if determined, blank otherwise + private string GetSourceFromFileName(string filename) + { + string source = ""; + + // Determine which dattype we have + GroupCollection fileinfo; + + if (Regex.IsMatch(filename, _nonGoodPattern)) + { + fileinfo = Regex.Match(filename, _nonGoodPattern).Groups; + if (!Remapping.NonGood.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped."); + return source; + } + source = "NonGood"; + } + else if (Regex.IsMatch(filename, _nonGoodSpecialPattern)) + { + fileinfo = Regex.Match(filename, _nonGoodSpecialPattern).Groups; + if (!Remapping.NonGood.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as NonGood but could not be mapped."); + return source; + } + source = "NonGood"; + } + else if (Regex.IsMatch(filename, _goodPattern)) + { + fileinfo = Regex.Match(filename, _goodPattern).Groups; + if (!Remapping.Good.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Good but could not be mapped."); + return source; + } + source = "Good"; + } + else if (Regex.IsMatch(filename, _goodXmlPattern)) + { + fileinfo = Regex.Match(filename, _goodXmlPattern).Groups; + } + else if (Regex.IsMatch(filename, _maybeIntroPattern)) + { + fileinfo = Regex.Match(filename, _maybeIntroPattern).Groups; + if (!Remapping.MaybeIntro.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Maybe-Intro but could not be mapped."); + return source; + } + source = "Maybe-Intro"; + } + else if (Regex.IsMatch(filename, _noIntroPattern)) + { + fileinfo = Regex.Match(filename, _noIntroPattern).Groups; + if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); + return source; + } + source = "no-Intro"; + } + // For numbered DATs only + else if (Regex.IsMatch(filename, _noIntroNumberedPattern)) + { + fileinfo = Regex.Match(filename, _noIntroNumberedPattern).Groups; + if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); + return source; + } + source = "no-Intro"; + } + // For N-Gage and Gizmondo only + else if (Regex.IsMatch(filename, _noIntroSpecialPattern)) + { + fileinfo = Regex.Match(filename, _noIntroSpecialPattern).Groups; + if (!Remapping.NoIntro.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as No-Intro but could not be mapped."); + return source; + } + source = "no-Intro"; + } + else if (Regex.IsMatch(filename, _redumpPattern)) + { + fileinfo = Regex.Match(filename, _redumpPattern).Groups; + if (!Remapping.Redump.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped."); + return source; + } + source = "Redump"; + } + // For special BIOSes only + else if (Regex.IsMatch(filename, _redumpBiosPattern)) + { + fileinfo = Regex.Match(filename, _redumpBiosPattern).Groups; + if (!Remapping.Redump.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as Redump but could not be mapped."); + return source; + } + source = "Redump"; + } + else if (Regex.IsMatch(filename, _tosecPattern)) + { + fileinfo = Regex.Match(filename, _tosecPattern).Groups; + if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) + { + // Handle special case mappings found only in TOSEC + fileinfo = Regex.Match(filename, _tosecSpecialPatternA).Groups; + + if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) + { + fileinfo = Regex.Match(filename, _tosecSpecialPatternB).Groups; + + if (!Remapping.TOSEC.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as TOSEC but could not be mapped."); + return source; + } + } + } + source = "TOSEC"; + } + else if (Regex.IsMatch(filename, _truripPattern)) + { + fileinfo = Regex.Match(filename, _truripPattern).Groups; + if (!Remapping.TruRip.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as TruRip but could not be mapped."); + return source; + } + source = "trurip"; + } + else if (Regex.IsMatch(filename, _zandroPattern)) + { + source = "Zandro"; + } + else if (Regex.IsMatch(filename, _defaultPattern)) + { + fileinfo = Regex.Match(filename, _defaultPattern).Groups; + source = fileinfo[3].Value; + } + else if (Regex.IsMatch(filename, _defaultSpecialPattern)) + { + fileinfo = Regex.Match(filename, _defaultSpecialPattern).Groups; + source = fileinfo[3].Value; + } + else if (Regex.IsMatch(filename, _mamePattern)) + { + fileinfo = Regex.Match(filename, _mamePattern).Groups; + if (!Remapping.MAME.ContainsKey(fileinfo[1].Value)) + { + _logger.Warning("The filename " + fileinfo[1].Value + " was matched as MAME but could not be mapped."); + return source; + } + source = "MAME"; + } + + return source; + } + } +}