using System; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; using System.Xml; using System.Xml.Schema; using SabreTools.Library.Data; using SabreTools.Library.Dats; using SabreTools.Library.External; using SabreTools.Library.Skippers; #if MONO using System.IO; #else using Alphaleonis.Win32.Filesystem; using BinaryReader = System.IO.BinaryReader; using BinaryWriter = System.IO.BinaryWriter; using FileAccess = System.IO.FileAccess; using FileMode = System.IO.FileMode; using FileShare = System.IO.FileShare; using FileStream = System.IO.FileStream; using IOException = System.IO.IOException; using MemoryStream = System.IO.MemoryStream; using PathTooLongException = System.IO.PathTooLongException; using SearchOption = System.IO.SearchOption; using SeekOrigin = System.IO.SeekOrigin; using Stream = System.IO.Stream; using StreamReader = System.IO.StreamReader; #endif using NaturalSort; using OCRC; namespace SabreTools.Library.Tools { public static class FileTools { #region File Information /// /// Get what type of DAT the input file is /// /// Name of the file to be parsed /// The DatFormat corresponding to the DAT /// There is currently no differentiation between XML and SabreDAT here public static DatFormat GetDatFormat(string filename) { // Limit the output formats based on extension string ext = Path.GetExtension(filename).ToLowerInvariant(); if (ext.StartsWith(".")) { ext = ext.Substring(1); } if (ext != "csv" && ext != "dat" && ext != "md5" && ext != "sfv" && ext != "sha1" && ext != "sha384" && ext != "sha512" && ext != "tsv" && ext != "txt" && ext != "xml") { return 0; } // Read the input file, if possible Globals.Logger.Verbose("Attempting to read file to get format: \"" + filename + "\""); // Check if file exists if (!File.Exists(filename)) { Globals.Logger.Warning("File '" + filename + "' could not read from!"); return 0; } // Some formats should only require the extension to know if (ext == "md5") { return DatFormat.RedumpMD5; } if (ext == "sfv") { return DatFormat.RedumpSFV; } if (ext == "sha1") { return DatFormat.RedumpSHA1; } if (ext == "sha256") { return DatFormat.RedumpSHA256; } if (ext == "sha384") { return DatFormat.RedumpSHA384; } if (ext == "sha512") { return DatFormat.RedumpSHA512; } if (ext == "csv") { return DatFormat.CSV; } if (ext == "tsv") { return DatFormat.TSV; } // For everything else, we need to read it try { // Get the first two lines to check StreamReader sr = File.OpenText(filename); string first = sr.ReadLine().ToLowerInvariant(); string second = sr.ReadLine().ToLowerInvariant(); sr.Dispose(); // If we have an XML-based DAT if (first.Contains("")) { if (second.StartsWith(" /// Get all empty folders within a root folder /// /// Root directory to parse /// IEumerable containing all directories that are empty, an empty enumerable if the root is empty, null otherwise public static IEnumerable GetEmptyDirectories(string root) { // Check if the root exists first if (!Directory.Exists(root)) { return null; } // If it does and it is empty, return a blank enumerable if (Directory.EnumerateFileSystemEntries(root, "*", SearchOption.AllDirectories).Count() == 0) { return new List(); } // Otherwise, get the complete list return Directory.EnumerateDirectories(root, "*", SearchOption.AllDirectories) .Where(dir => Directory.EnumerateFileSystemEntries(dir, "*", SearchOption.AllDirectories).Count() == 0); } /// /// Retrieve file information for a single file /// /// Filename to get information from /// Hash flag saying what hashes should not be calculated (defaults to none) /// Set a >0 number for getting hash for part of the file, 0 otherwise (default) /// True if the file Date should be included, false otherwise (default) /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Populated RomData object if success, empty one on error public static Rom GetFileInfo(string input, Hash omitFromScan = 0x0, long offset = 0, bool date = false, string header = null) { // Add safeguard if file doesn't exist if (!File.Exists(input)) { return new Rom(); } // Get the information from the file stream Rom rom = new Rom(); if (header != null) { SkipperRule rule = Skipper.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header)); // If there's a match, get the new information from the stream if (rule.Tests != null && rule.Tests.Count != 0) { // Create the input and output streams MemoryStream outputStream = new MemoryStream(); FileStream inputStream = FileTools.TryOpenRead(input); // Transform the stream and get the information from it rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true); rom = GetStreamInfo(outputStream, outputStream.Length, omitFromScan: omitFromScan, keepReadOpen: false); // Dispose of the streams outputStream.Dispose(); inputStream.Dispose(); } // Otherwise, just get the info else { long length = new FileInfo(input).Length; rom = GetStreamInfo(TryOpenRead(input), length, omitFromScan, offset, false); } } else { long length = new FileInfo(input).Length; rom = GetStreamInfo(TryOpenRead(input), length, omitFromScan, offset, false); } // Add unique data from the file rom.Name = Path.GetFileName(input); rom.Date = (date ? new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss") : ""); return rom; } /// /// Retrieve a list of files from a directory recursively in proper order /// /// Directory to parse /// List representing existing files /// List with all new files public static List RetrieveFiles(string directory, List infiles) { // Take care of the files in the top directory List toadd = Directory.EnumerateFiles(directory, "*", SearchOption.TopDirectoryOnly).ToList(); toadd.Sort(new NaturalComparer()); infiles.AddRange(toadd); // Then recurse through and add from the directories List dirs = Directory.EnumerateDirectories(directory, "*", SearchOption.TopDirectoryOnly).ToList(); dirs.Sort(new NaturalComparer()); foreach (string dir in dirs) { infiles = RetrieveFiles(dir, infiles); } // Return the new list return infiles; } #endregion #region File Manipulation /// /// Add an aribtrary number of bytes to the inputted file /// /// File to be appended to /// Outputted file /// String representing bytes to be added to head of file /// String representing bytes to be added to tail of file public static void AppendBytesToFile(string input, string output, string bytesToAddToHead, string bytesToAddToTail) { // Source: http://stackoverflow.com/questions/311165/how-do-you-convert-byte-array-to-hexadecimal-string-and-vice-versa byte[] bytesToAddToHeadArray = new byte[bytesToAddToHead.Length / 2]; for (int i = 0; i < bytesToAddToHead.Length; i += 2) { bytesToAddToHeadArray[i / 2] = Convert.ToByte(bytesToAddToHead.Substring(i, 2), 16); } byte[] bytesToAddToTailArray = new byte[bytesToAddToTail.Length / 2]; for (int i = 0; i < bytesToAddToTail.Length; i += 2) { bytesToAddToTailArray[i / 2] = Convert.ToByte(bytesToAddToTail.Substring(i, 2), 16); } AppendBytesToFile(input, output, bytesToAddToHeadArray, bytesToAddToTailArray); } /// /// Add an aribtrary number of bytes to the inputted file /// /// File to be appended to /// Outputted file /// Bytes to be added to head of file /// Bytes to be added to tail of file public static void AppendBytesToFile(string input, string output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) { // If any of the inputs are invalid, skip if (!File.Exists(input)) { return; } FileStream fsr = TryOpenRead(input); FileStream fsw = TryOpenWrite(output); AppendBytesToStream(fsr, fsw, bytesToAddToHead, bytesToAddToTail); fsr.Dispose(); fsw.Dispose(); } /// /// Cleans out the temporary directory /// /// Name of the directory to clean out public static void CleanDirectory(string dirname) { foreach (string file in Directory.EnumerateFiles(dirname, "*", SearchOption.TopDirectoryOnly)) { TryDeleteFile(file); } foreach (string dir in Directory.EnumerateDirectories(dirname, "*", SearchOption.TopDirectoryOnly)) { TryDeleteDirectory(dir); } } /// /// Detect header skipper compliance and create an output file /// /// Name of the file to be parsed /// Output directory to write the file to, empty means the same directory as the input file /// True if the output file was created, false otherwise public static bool DetectSkipperAndTransform(string file, string outDir) { // Create the output directory if it doesn't exist if (outDir != "" && !Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); } Globals.Logger.User("\nGetting skipper information for '" + file + "'"); // Get the skipper rule that matches the file, if any SkipperRule rule = Skipper.GetMatchingRule(file, ""); // If we have an empty rule, return false if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None) { return false; } Globals.Logger.User("File has a valid copier header"); // Get the header bytes from the file first string hstr = string.Empty; BinaryReader br = new BinaryReader(TryOpenRead(file)); // Extract the header as a string for the database byte[] hbin = br.ReadBytes((int)rule.StartOffset); for (int i = 0; i < (int)rule.StartOffset; i++) { hstr += BitConverter.ToString(new byte[] { hbin[i] }); } br.Dispose(); // Apply the rule to the file string newfile = (outDir == "" ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file))); rule.TransformFile(file, newfile); // If the output file doesn't exist, return false if (!File.Exists(newfile)) { return false; } // Now add the information to the database if it's not already there Rom rom = GetFileInfo(newfile); DatabaseTools.AddHeaderToDatabase(hstr, rom.SHA1, rule.SourceFile); return true; } /// /// Retrieve a list of just files from inputs /// /// List of strings representing directories and files /// True if the parent name should be appended after the special character "¬", false otherwise (default) /// List of strings representing just files from the inputs public static List GetOnlyFilesFromInputs(List inputs, bool appendparent = false) { List outputs = new List(); foreach (string input in inputs) { if (Directory.Exists(input)) { List files = FileTools.RetrieveFiles(input, new List()); foreach (string file in files) { try { outputs.Add(Path.GetFullPath(file) + (appendparent ? "¬" + Path.GetFullPath(input) : "")); } catch (PathTooLongException) { Globals.Logger.Warning("The path for " + file + " was too long"); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); } } } else if (File.Exists(input)) { try { outputs.Add(Path.GetFullPath(input) + (appendparent ? "¬" + Path.GetFullPath(input) : "")); } catch (PathTooLongException) { Globals.Logger.Warning("The path for " + input + " was too long"); } catch (Exception ex) { Globals.Logger.Error(ex.ToString()); } } } return outputs; } /// /// Get the XmlTextReader associated with a file, if possible /// /// Name of the file to be parsed /// The XmlTextReader representing the (possibly converted) file, null otherwise public static XmlReader GetXmlTextReader(string filename) { Globals.Logger.Verbose("Attempting to read file: \"" + filename + "\""); // Check if file exists if (!File.Exists(filename)) { Globals.Logger.Warning("File '" + filename + "' could not read from!"); return null; } XmlReader xtr = XmlReader.Create(filename, new XmlReaderSettings { CheckCharacters = false, DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true, IgnoreWhitespace = true, ValidationFlags = XmlSchemaValidationFlags.None, ValidationType = ValidationType.None, }); return xtr; } /// /// Detect and replace header(s) to the given file /// /// Name of the file to be parsed /// Output directory to write the file to, empty means the same directory as the input file /// True if a header was found and appended, false otherwise public static bool RestoreHeader(string file, string outDir) { // Create the output directory if it doesn't exist if (outDir != "" && !Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); } // First, get the SHA-1 hash of the file Rom rom = GetFileInfo(file); // Retrieve a list of all related headers from the database List headers = DatabaseTools.RetrieveHeadersFromDatabase(rom.SHA1); // If we have nothing retrieved, we return false if (headers.Count == 0) { return false; } // Now loop through and create the reheadered files, if possible for (int i = 0; i < headers.Count; i++) { Globals.Logger.User("Creating reheadered file: " + (outDir == "" ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file))) + i); AppendBytesToFile(file, (outDir == "" ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file))) + i, headers[i], string.Empty); Globals.Logger.User("Reheadered file created!"); } return true; } /// /// Try to create a file for write, optionally throwing the error /// /// Name of the file to create /// True if the error that is thrown should be thrown back to the caller, false otherwise /// An opened stream representing the file on success, null otherwise public static FileStream TryCreate(string file, bool throwOnError = false) { // Now wrap opening the file try { return File.Open(file, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); } catch (Exception ex) { if (throwOnError) { throw ex; } else { return null; } } } /// /// Try to safely delete a directory, optionally throwing the error /// /// Name of the directory to delete /// True if the error that is thrown should be thrown back to the caller, false otherwise /// True if the file didn't exist or could be deleted, false otherwise public static bool TryDeleteDirectory(string file, bool throwOnError = false) { // Check if the file exists first if (!Directory.Exists(file)) { return true; } // Now wrap deleting the file try { Directory.Delete(file, true); return true; } catch (Exception ex) { if (throwOnError) { throw ex; } else { return false; } } } /// /// Try to safely delete a file, optionally throwing the error /// /// Name of the file to delete /// True if the error that is thrown should be thrown back to the caller, false otherwise /// True if the file didn't exist or could be deleted, false otherwise public static bool TryDeleteFile(string file, bool throwOnError = false) { // Check if the file exists first if (!File.Exists(file)) { return true; } // Now wrap deleting the file try { File.Delete(file); return true; } catch (Exception ex) { if (throwOnError) { throw ex; } else { return false; } } } /// /// Try to open a file for read, optionally throwing the error /// /// Name of the file to open /// True if the error that is thrown should be thrown back to the caller, false otherwise /// An opened stream representing the file on success, null otherwise public static FileStream TryOpenRead(string file, bool throwOnError = false) { // Check if the file exists first if (!File.Exists(file)) { return null; } // Now wrap opening the file try { return File.Open(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); } catch (Exception ex) { if (throwOnError) { throw ex; } else { return null; } } } /// /// Try to open a file for read/write, optionally throwing the error /// /// Name of the file to open /// True if the error that is thrown should be thrown back to the caller, false otherwise /// An opened stream representing the file on success, null otherwise public static FileStream TryOpenReadWrite(string file, bool throwOnError = false) { // Check if the file exists first if (!File.Exists(file)) { return null; } // Now wrap opening the file try { return File.Open(file, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); } catch (Exception ex) { if (throwOnError) { throw ex; } else { return null; } } } /// /// Try to open a file for write, optionally throwing the error /// /// Name of the file to open /// True if the error that is thrown should be thrown back to the caller, false otherwise /// An opened stream representing the file on success, null otherwise public static FileStream TryOpenWrite(string file, bool throwOnError = false) { // Check if the file exists first if (!File.Exists(file)) { return null; } // Now wrap opening the file try { return File.Open(file, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); } catch (Exception ex) { if (throwOnError) { throw ex; } else { return null; } } } #endregion #region Stream Information /// /// Retrieve file information for a single file /// /// Filename to get information from /// Size of the input stream /// Hash flag saying what hashes should not be calculated (defaults to none) /// Set a >0 number for getting hash for part of the file, 0 otherwise (default) /// True if the underlying read stream should be kept open, false otherwise /// Populated RomData object if success, empty one on error public static Rom GetStreamInfo(Stream input, long size, Hash omitFromScan = 0x0, long offset = 0, bool keepReadOpen = false) { Rom rom = new Rom { Type = ItemType.Rom, Size = size, CRC = string.Empty, MD5 = string.Empty, SHA1 = string.Empty, SHA256 = string.Empty, SHA384 = string.Empty, SHA512 = string.Empty, }; try { // Initialize the hashers OptimizedCRC crc = new OptimizedCRC(); MD5 md5 = MD5.Create(); SHA1 sha1 = SHA1.Create(); SHA256 sha256 = SHA256.Create(); SHA384 sha384 = SHA384.Create(); SHA512 sha512 = SHA512.Create(); xxHash xxHash = new xxHash(); xxHash.Init(); // Seek to the starting position, if one is set if (offset < 0) { input.Seek(offset, SeekOrigin.End); } else { input.Seek(offset, SeekOrigin.Begin); } byte[] buffer = new byte[8 * 1024]; int read; while ((read = input.Read(buffer, 0, buffer.Length)) > 0) { crc.Update(buffer, 0, read); if ((omitFromScan & Hash.MD5) == 0) { md5.TransformBlock(buffer, 0, read, buffer, 0); } if ((omitFromScan & Hash.SHA1) == 0) { sha1.TransformBlock(buffer, 0, read, buffer, 0); } if ((omitFromScan & Hash.SHA256) == 0) { sha256.TransformBlock(buffer, 0, read, buffer, 0); } if ((omitFromScan & Hash.SHA384) == 0) { sha384.TransformBlock(buffer, 0, read, buffer, 0); } if ((omitFromScan & Hash.SHA512) == 0) { sha512.TransformBlock(buffer, 0, read, buffer, 0); } if ((omitFromScan & Hash.xxHash) == 0) { xxHash.Update(buffer, read); } } crc.Update(buffer, 0, 0); rom.CRC = crc.Value.ToString("X8").ToLowerInvariant(); if ((omitFromScan & Hash.MD5) == 0) { md5.TransformFinalBlock(buffer, 0, 0); rom.MD5 = BitConverter.ToString(md5.Hash).Replace("-", "").ToLowerInvariant(); } if ((omitFromScan & Hash.SHA1) == 0) { sha1.TransformFinalBlock(buffer, 0, 0); rom.SHA1 = BitConverter.ToString(sha1.Hash).Replace("-", "").ToLowerInvariant(); } if ((omitFromScan & Hash.SHA256) == 0) { sha256.TransformFinalBlock(buffer, 0, 0); rom.SHA256 = BitConverter.ToString(sha256.Hash).Replace("-", "").ToLowerInvariant(); } if ((omitFromScan & Hash.SHA384) == 0) { sha384.TransformFinalBlock(buffer, 0, 0); rom.SHA384 = BitConverter.ToString(sha384.Hash).Replace("-", "").ToLowerInvariant(); } if ((omitFromScan & Hash.SHA512) == 0) { sha512.TransformFinalBlock(buffer, 0, 0); rom.SHA512 = BitConverter.ToString(sha512.Hash).Replace("-", "").ToLowerInvariant(); } if ((omitFromScan & Hash.xxHash) == 0) { //rom.xxHash = xxHash.Digest().ToString("X8").ToLowerInvariant(); } // Dispose of the hashers crc.Dispose(); md5.Dispose(); sha1.Dispose(); sha256.Dispose(); sha384.Dispose(); sha512.Dispose(); } catch (IOException) { return new Rom(); } finally { // Seek to the beginning of the stream input.Seek(0, SeekOrigin.Begin); if (!keepReadOpen) { input.Dispose(); } } return rom; } #endregion #region Stream Manipulation /// /// Add an aribtrary number of bytes to the inputted stream /// /// Stream to be appended to /// Outputted stream /// Bytes to be added to head of stream /// Bytes to be added to tail of stream public static void AppendBytesToStream(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) { BinaryReader br = new BinaryReader(input); BinaryWriter bw = new BinaryWriter(output); if (bytesToAddToHead.Count() > 0) { bw.Write(bytesToAddToHead); } int bufferSize = 1024; // Now read the file in chunks and write out byte[] buffer = new byte[bufferSize]; while (br.BaseStream.Position <= (br.BaseStream.Length - bufferSize)) { buffer = br.ReadBytes(bufferSize); bw.Write(buffer); } // For the final chunk, if any, write out only that number of bytes int length = (int)(br.BaseStream.Length - br.BaseStream.Position); buffer = new byte[length]; buffer = br.ReadBytes(length); bw.Write(buffer); if (bytesToAddToTail.Count() > 0) { bw.Write(bytesToAddToTail); } } #endregion } }