diff --git a/RombaSharp/Features/Archive.cs b/RombaSharp/Features/Archive.cs index ee6e8c6a..4e0f16ff 100644 --- a/RombaSharp/Features/Archive.cs +++ b/RombaSharp/Features/Archive.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; +using SabreTools.Data; using SabreTools.Help; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; diff --git a/RombaSharp/Features/BaseFeature.cs b/RombaSharp/Features/BaseFeature.cs index 4649b9df..6a46fd43 100644 --- a/RombaSharp/Features/BaseFeature.cs +++ b/RombaSharp/Features/BaseFeature.cs @@ -3,13 +3,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Xml; +using System.Xml.Schema; using SabreTools.Data; using SabreTools.Help; using SabreTools.Logging; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; -using SabreTools.Library.IO; +using SabreTools.Library.FileTypes; using SabreTools.Library.Tools; using Microsoft.Data.Sqlite; @@ -479,7 +480,7 @@ CREATE TABLE IF NOT EXISTS dat ( if (lowerCaseDats.Contains(input.ToLowerInvariant())) { string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]); - string sha1 = Utilities.ByteArrayToString(FileExtensions.GetInfo(fullpath, hashes: Hash.SHA1).SHA1); + string sha1 = Utilities.ByteArrayToString(BaseFile.GetInfo(fullpath, hashes: Hash.SHA1).SHA1); foundDats.Add(sha1, fullpath); } else @@ -510,7 +511,15 @@ CREATE TABLE IF NOT EXISTS dat ( Dictionary> depots = new Dictionary>(); // Get the XML text reader for the configuration file, if possible - XmlReader xtr = _config.GetXmlTextReader(); + XmlReader xtr = XmlReader.Create(_config, new XmlReaderSettings + { + CheckCharacters = false, + DtdProcessing = DtdProcessing.Ignore, + IgnoreComments = true, + IgnoreWhitespace = true, + ValidationFlags = XmlSchemaValidationFlags.None, + ValidationType = ValidationType.None, + }); // Now parse the XML file for settings if (xtr != null) diff --git a/RombaSharp/Features/Export.cs b/RombaSharp/Features/Export.cs index 058f814a..ca55cd43 100644 --- a/RombaSharp/Features/Export.cs +++ b/RombaSharp/Features/Export.cs @@ -2,7 +2,6 @@ using System.IO; using SabreTools.Help; -using SabreTools.Library.IO; using Microsoft.Data.Sqlite; namespace RombaSharp.Features @@ -29,7 +28,7 @@ namespace RombaSharp.Features SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); - StreamWriter sw = new StreamWriter(FileExtensions.TryCreate("export.csv")); + StreamWriter sw = new StreamWriter(File.Create("export.csv")); // First take care of all file hashes sw.WriteLine("CRC,MD5,SHA-1"); // ,Depot diff --git a/RombaSharp/Features/Import.cs b/RombaSharp/Features/Import.cs index bb6184c4..3e25d566 100644 --- a/RombaSharp/Features/Import.cs +++ b/RombaSharp/Features/Import.cs @@ -4,7 +4,6 @@ using System.Linq; using SabreTools.Help; using SabreTools.IO; -using SabreTools.Library.IO; using Microsoft.Data.Sqlite; namespace RombaSharp.Features @@ -38,7 +37,7 @@ namespace RombaSharp.Features // Now, for each of these files, attempt to add the data found inside foreach (string input in Inputs) { - StreamReader sr = new StreamReader(FileExtensions.TryOpenRead(input)); + StreamReader sr = new StreamReader(File.OpenRead(input)); // The first line should be the hash header string line = sr.ReadLine(); diff --git a/RombaSharp/Features/RescanDepots.cs b/RombaSharp/Features/RescanDepots.cs index e175f43f..af1d4259 100644 --- a/RombaSharp/Features/RescanDepots.cs +++ b/RombaSharp/Features/RescanDepots.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; +using SabreTools.Data; using SabreTools.Help; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; diff --git a/SabreTools.Data/Enums.cs b/SabreTools.Data/Enums.cs index c12bf9de..2227f078 100644 --- a/SabreTools.Data/Enums.cs +++ b/SabreTools.Data/Enums.cs @@ -2,6 +2,166 @@ namespace SabreTools.Data { + #region DatFile + + /// + /// DAT output formats + /// + [Flags] + public enum DatFormat + { + #region XML Formats + + /// + /// Logiqx XML (using machine) + /// + Logiqx = 1 << 0, + + /// + /// Logiqx XML (using game) + /// + LogiqxDeprecated = 1 << 1, + + /// + /// MAME Softare List XML + /// + SoftwareList = 1 << 2, + + /// + /// MAME Listxml output + /// + Listxml = 1 << 3, + + /// + /// OfflineList XML + /// + OfflineList = 1 << 4, + + /// + /// SabreDAT XML + /// + SabreXML = 1 << 5, + + /// + /// openMSX Software List XML + /// + OpenMSX = 1 << 6, + + #endregion + + #region Propietary Formats + + /// + /// ClrMamePro custom + /// + ClrMamePro = 1 << 7, + + /// + /// RomCenter INI-based + /// + RomCenter = 1 << 8, + + /// + /// DOSCenter custom + /// + DOSCenter = 1 << 9, + + /// + /// AttractMode custom + /// + AttractMode = 1 << 10, + + #endregion + + #region Standardized Text Formats + + /// + /// ClrMamePro missfile + /// + MissFile = 1 << 11, + + /// + /// Comma-Separated Values (standardized) + /// + CSV = 1 << 12, + + /// + /// Semicolon-Separated Values (standardized) + /// + SSV = 1 << 13, + + /// + /// Tab-Separated Values (standardized) + /// + TSV = 1 << 14, + + /// + /// MAME Listrom output + /// + Listrom = 1 << 15, + + /// + /// Everdrive Packs SMDB + /// + EverdriveSMDB = 1 << 16, + + /// + /// SabreJSON + /// + SabreJSON = 1 << 17, + + #endregion + + #region SFV-similar Formats + + /// + /// CRC32 hash list + /// + RedumpSFV = 1 << 18, + + /// + /// MD5 hash list + /// + RedumpMD5 = 1 << 19, + +#if NET_FRAMEWORK + /// + /// RIPEMD160 hash list + /// + RedumpRIPEMD160 = 1 << 20, +#endif + + /// + /// SHA-1 hash list + /// + RedumpSHA1 = 1 << 21, + + /// + /// SHA-256 hash list + /// + RedumpSHA256 = 1 << 22, + + /// + /// SHA-384 hash list + /// + RedumpSHA384 = 1 << 23, + + /// + /// SHA-512 hash list + /// + RedumpSHA512 = 1 << 24, + + /// + /// SpamSum hash list + /// + RedumpSpamSum = 1 << 25, + + #endregion + + // Specialty combinations + ALL = Int32.MaxValue, + } + /// /// Available hashing types /// @@ -29,4 +189,49 @@ namespace SabreTools.Data SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, #endif } + + /// + /// Determines what sort of files get externally hashed + /// + /// TODO: Can FileType be used instead? + [Flags] + public enum TreatAsFile + { + CHD = 1 << 0, + Archive = 1 << 1, + AaruFormat = 1 << 2, + + NonArchive = CHD | AaruFormat, + All = CHD | Archive | AaruFormat, + } + + #endregion + + #region FileTypes + + /// + /// Type of file that is being looked at + /// + public enum FileType + { + // Singleton + None = 0, + AaruFormat, + CHD, + + // Can contain children + Folder, + SevenZipArchive, + GZipArchive, + LRZipArchive, + LZ4Archive, + RarArchive, + TapeArchive, + XZArchive, + ZipArchive, + ZPAQArchive, + ZstdArchive, + } + + #endregion } diff --git a/SabreTools.IO/DirectoryExtensions.cs b/SabreTools.IO/DirectoryExtensions.cs index 366b2aed..5e240696 100644 --- a/SabreTools.IO/DirectoryExtensions.cs +++ b/SabreTools.IO/DirectoryExtensions.cs @@ -3,8 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using SabreTools.Logging; - +// TODO: Figure out a reasonable way of adding logging back to this namespace SabreTools.IO { /// @@ -20,12 +19,22 @@ namespace SabreTools.IO { foreach (string file in Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly)) { - FileTryDelete(file); + try + { + if (File.Exists(file)) + File.Delete(file); + } + catch { } } foreach (string subdir in Directory.EnumerateDirectories(dir, "*", SearchOption.TopDirectoryOnly)) { - TryDelete(subdir); + try + { + if (Directory.Exists(subdir)) + Directory.Delete(subdir); + } + catch { } } } @@ -90,7 +99,7 @@ namespace SabreTools.IO } catch (Exception ex) { - LoggerImpl.Error(ex, $"An exception occurred getting the full path for '{input}'"); + //LoggerImpl.Error(ex, $"An exception occurred getting the full path for '{input}'"); continue; } @@ -105,11 +114,11 @@ namespace SabreTools.IO } catch (PathTooLongException ex) { - LoggerImpl.Warning(ex, $"The path for '{dir}' was too long"); + //LoggerImpl.Warning(ex, $"The path for '{dir}' was too long"); } catch (Exception ex) { - LoggerImpl.Error(ex, $"An exception occurred processing '{dir}'"); + //LoggerImpl.Error(ex, $"An exception occurred processing '{dir}'"); } } } @@ -186,7 +195,7 @@ namespace SabreTools.IO } catch (Exception ex) { - LoggerImpl.Error(ex, $"An exception occurred getting the full path for '{input}'"); + //LoggerImpl.Error(ex, $"An exception occurred getting the full path for '{input}'"); continue; } @@ -201,11 +210,11 @@ namespace SabreTools.IO } catch (PathTooLongException ex) { - LoggerImpl.Warning(ex, $"The path for '{file}' was too long"); + //LoggerImpl.Warning(ex, $"The path for '{file}' was too long"); } catch (Exception ex) { - LoggerImpl.Error(ex, $"An exception occurred processing '{file}'"); + //LoggerImpl.Error(ex, $"An exception occurred processing '{file}'"); } } } @@ -217,11 +226,11 @@ namespace SabreTools.IO } catch (PathTooLongException ex) { - LoggerImpl.Warning(ex, $"The path for '{input}' was too long"); + //LoggerImpl.Warning(ex, $"The path for '{input}' was too long"); } catch (Exception ex) { - LoggerImpl.Error(ex, $"An exception occurred processing '{input}'"); + //LoggerImpl.Error(ex, $"An exception occurred processing '{input}'"); } } } @@ -287,86 +296,9 @@ namespace SabreTools.IO .ToList(); } - /// - /// Try to safely delete a directory, optionally throwing the error - /// - /// Name of the directory to delete - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the file didn't exist or could be deleted, false otherwise - public static bool TryCreateDirectory(string file, bool throwOnError = false) - { - // Now wrap creating the directory - try - { - Directory.CreateDirectory(file); - return true; - } - catch (Exception ex) - { - if (throwOnError) - throw ex; - else - return false; - } - } - - /// - /// Try to safely delete a directory, optionally throwing the error - /// - /// Name of the directory to delete - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the file didn't exist or could be deleted, false otherwise - public static bool TryDelete(string file, bool throwOnError = false) - { - // Check if the directory exists first - if (!Directory.Exists(file)) - return true; - - // Now wrap deleting the directory - try - { - Directory.Delete(file, true); - return true; - } - catch (Exception ex) - { - if (throwOnError) - throw ex; - else - return false; - } - } - // TODO: Remove this entire section once External and the rest of IO is in its own library (or pulled in otherwise) #region TEMPORARY - REMOVEME - /// - /// Try to safely delete a file, optionally throwing the error - /// - /// Name of the file to delete - /// True if the error that is thrown should be thrown back to the caller, false otherwise - /// True if the file didn't exist or could be deleted, false otherwise - private static bool FileTryDelete(string file, bool throwOnError = false) - { - // Check if the file exists first - if (!File.Exists(file)) - return true; - - // Now wrap deleting the file - try - { - File.Delete(file); - return true; - } - catch (Exception ex) - { - if (throwOnError) - throw ex; - else - return false; - } - } - private class NaturalComparer : Comparer, IDisposable { private Dictionary table; diff --git a/SabreTools.IO/FileExtensions.cs b/SabreTools.IO/FileExtensions.cs new file mode 100644 index 00000000..be9a15fa --- /dev/null +++ b/SabreTools.IO/FileExtensions.cs @@ -0,0 +1,77 @@ +using System.IO; +using System.Text; + +namespace SabreTools.IO +{ + /// + /// Extensions to File functionality + /// + public static class FileExtensions + { + /// + /// Determines a text file's encoding by analyzing its byte order mark (BOM). + /// Defaults to ASCII when detection of the text file's endianness fails. + /// + /// The text file to analyze. + /// The detected encoding. + /// http://stackoverflow.com/questions/3825390/effective-way-to-find-any-files-encoding + public static Encoding GetEncoding(string filename) + { + if (!File.Exists(filename)) + return Encoding.Default; + + // Try to open the file + try + { + FileStream file = File.OpenRead(filename); + if (file == null) + return Encoding.Default; + + // Read the BOM + var bom = new byte[4]; + file.Read(bom, 0, 4); + file.Dispose(); + + // Analyze the BOM + if (bom[0] == 0x2b && bom[1] == 0x2f && bom[2] == 0x76) return Encoding.UTF7; + if (bom[0] == 0xef && bom[1] == 0xbb && bom[2] == 0xbf) return Encoding.UTF8; + if (bom[0] == 0xff && bom[1] == 0xfe) return Encoding.Unicode; //UTF-16LE + if (bom[0] == 0xfe && bom[1] == 0xff) return Encoding.BigEndianUnicode; //UTF-16BE + if (bom[0] == 0 && bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff) return Encoding.UTF32; + return Encoding.Default; + } + catch + { + return Encoding.Default; + } + } + + /// + /// Returns if the first byte array starts with the second array + /// + /// First byte array to compare + /// Second byte array to compare + /// True if the input arrays should match exactly, false otherwise (default) + /// True if the first byte array starts with the second, false otherwise + public static bool StartsWith(this byte[] arr1, byte[] arr2, bool exact = false) + { + // If we have any invalid inputs, we return false + if (arr1 == null || arr2 == null + || arr1.Length == 0 || arr2.Length == 0 + || arr2.Length > arr1.Length + || (exact && arr1.Length != arr2.Length)) + { + return false; + } + + // Otherwise, loop through and see + for (int i = 0; i < arr2.Length; i++) + { + if (arr1[i] != arr2[i]) + return false; + } + + return true; + } + } +} diff --git a/SabreTools.IO/PathExtensions.cs b/SabreTools.IO/PathExtensions.cs index 862285b0..8b837644 100644 --- a/SabreTools.IO/PathExtensions.cs +++ b/SabreTools.IO/PathExtensions.cs @@ -2,7 +2,7 @@ using SabreTools.Data; -namespace SabreTools.Library.IO +namespace SabreTools.IO { /// /// Extensions to Path functionality diff --git a/SabreTools.IO/SabreTools.IO.csproj b/SabreTools.IO/SabreTools.IO.csproj index d076577d..52f3272d 100644 --- a/SabreTools.IO/SabreTools.IO.csproj +++ b/SabreTools.IO/SabreTools.IO.csproj @@ -13,7 +13,6 @@ - diff --git a/SabreTools.IO/StreamExtensions.cs b/SabreTools.IO/StreamExtensions.cs index a56c7c3d..985003f3 100644 --- a/SabreTools.IO/StreamExtensions.cs +++ b/SabreTools.IO/StreamExtensions.cs @@ -1,9 +1,6 @@ -using System; -using System.IO; +using System.IO; using System.Linq; -using SabreTools.Logging; - namespace SabreTools.IO { /// @@ -11,27 +8,6 @@ namespace SabreTools.IO /// public static class StreamExtensions { - /// - /// Add an aribtrary number of bytes to the inputted stream - /// - /// Stream to be appended to - /// Outputted stream - /// Bytes to be added to head of stream - /// Bytes to be added to tail of stream - public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) - { - // Write out prepended bytes - if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0) - output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length); - - // Now copy the existing file over - input.CopyTo(output); - - // Write out appended bytes - if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0) - output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length); - } - /// /// Seek to a specific point in the stream, if possible /// @@ -51,16 +27,10 @@ namespace SabreTools.IO return input.Position; } - catch (NotSupportedException ex) + catch { - LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed"); + return -1; } - catch (NotImplementedException ex) - { - LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed"); - } - - return -1; } } } diff --git a/SabreTools.Library/DatFiles/AttractMode.cs b/SabreTools.Library/DatFiles/AttractMode.cs index e3983560..a0fce09e 100644 --- a/SabreTools.Library/DatFiles/AttractMode.cs +++ b/SabreTools.Library/DatFiles/AttractMode.cs @@ -35,7 +35,7 @@ namespace SabreTools.Library.DatFiles { // Open a file reader Encoding enc = FileExtensions.GetEncoding(filename); - SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc) + SeparatedValueReader svr = new SeparatedValueReader(File.OpenRead(filename), enc) { Header = true, Quotes = false, @@ -134,7 +134,7 @@ namespace SabreTools.Library.DatFiles try { logger.User($"Opening file for writing: {outfile}"); - FileStream fs = FileExtensions.TryCreate(outfile); + FileStream fs = File.Create(outfile); // If we get back null for some reason, just log and return if (fs == null) diff --git a/SabreTools.Library/DatFiles/ClrMamePro.cs b/SabreTools.Library/DatFiles/ClrMamePro.cs index cf8b8595..31863e8d 100644 --- a/SabreTools.Library/DatFiles/ClrMamePro.cs +++ b/SabreTools.Library/DatFiles/ClrMamePro.cs @@ -46,7 +46,7 @@ namespace SabreTools.Library.DatFiles { // Open a file reader Encoding enc = FileExtensions.GetEncoding(filename); - ClrMameProReader cmpr = new ClrMameProReader(FileExtensions.TryOpenRead(filename), enc) + ClrMameProReader cmpr = new ClrMameProReader(File.OpenRead(filename), enc) { DosCenter = false, Quotes = Quotes, @@ -459,7 +459,7 @@ namespace SabreTools.Library.DatFiles try { logger.User($"Opening file for writing: {outfile}"); - FileStream fs = FileExtensions.TryCreate(outfile); + FileStream fs = File.Create(outfile); // If we get back null for some reason, just log and return if (fs == null) diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index 5223b218..f93a8da4 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -17,8 +17,8 @@ using SabreTools.Library.FileTypes; using SabreTools.Library.Filtering; using SabreTools.Library.IO; using SabreTools.Library.Reports; -using SabreTools.Library.Skippers; using SabreTools.Library.Tools; +using SabreTools.Skippers; using NaturalSort; using Newtonsoft.Json; @@ -1870,13 +1870,14 @@ namespace SabreTools.Library.DatFiles Header.FileName = (string.IsNullOrWhiteSpace(Header.FileName) ? (keepext ? Path.GetFileName(currentPath) : Path.GetFileNameWithoutExtension(currentPath)) : Header.FileName); // If the output type isn't set already, get the internal output type - Header.DatFormat = (Header.DatFormat == 0 ? currentPath.GetDatFormat() : Header.DatFormat); + DatFormat currentPathFormat = GetDatFormat(currentPath); + Header.DatFormat = (Header.DatFormat == 0 ? currentPathFormat : Header.DatFormat); Items.SetBucketedBy(Field.DatItem_CRC); // Setting this because it can reduce issues later // Now parse the correct type of DAT try { - Create(currentPath.GetDatFormat(), this, quotes)?.ParseFile(currentPath, indexId, keep, throwOnError); + Create(currentPathFormat, this, quotes)?.ParseFile(currentPath, indexId, keep, throwOnError); } catch (Exception ex) { @@ -1885,6 +1886,142 @@ namespace SabreTools.Library.DatFiles } } + /// + /// Get what type of DAT the input file is + /// + /// Name of the file to be parsed + /// The DatFormat corresponding to the DAT + protected DatFormat GetDatFormat(string filename) + { + // Limit the output formats based on extension + if (!PathExtensions.HasValidDatExtension(filename)) + return 0; + + // Get the extension from the filename + string ext = PathExtensions.GetNormalizedExtension(filename); + + // Check if file exists + if (!File.Exists(filename)) + return 0; + + // Some formats should only require the extension to know + switch (ext) + { + case "csv": + return DatFormat.CSV; + case "json": + return DatFormat.SabreJSON; + case "md5": + return DatFormat.RedumpMD5; +#if NET_FRAMEWORK + case "ripemd160": + return DatFormat.RedumpRIPEMD160; +#endif + case "sfv": + return DatFormat.RedumpSFV; + case "sha1": + return DatFormat.RedumpSHA1; + case "sha256": + return DatFormat.RedumpSHA256; + case "sha384": + return DatFormat.RedumpSHA384; + case "sha512": + return DatFormat.RedumpSHA512; + case "spamsum": + return DatFormat.RedumpSpamSum; + case "ssv": + return DatFormat.SSV; + case "tsv": + return DatFormat.TSV; + } + + // For everything else, we need to read it + // Get the first two non-whitespace, non-comment lines to check, if possible + string first = string.Empty, second = string.Empty; + + try + { + using (StreamReader sr = File.OpenText(filename)) + { + first = sr.ReadLine().ToLowerInvariant(); + while ((string.IsNullOrWhiteSpace(first) || first.StartsWith("