using SabreTools.Library.Data; using SabreTools.Library.FileTypes; using SabreTools.Library.Tools; using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Xml; namespace SabreTools.Library.Skippers { public class Skipper { #region Fields /// /// Skipper name /// public string Name { get; set; } /// /// Author names /// public string Author { get; set; } /// /// File version /// public string Version { get; set; } /// /// Set of all rules in the skipper /// public List Rules { get; set; } /// /// Filename the skipper lives in /// public string SourceFile { get; set; } /// /// Local paths /// public static string LocalPath = Path.Combine(Globals.ExeDir, "Skippers") + Path.DirectorySeparatorChar; /// /// Header skippers represented by a list of skipper objects /// private static List List; #endregion #region Constructors /// /// Create an empty Skipper object /// public Skipper() { Name = string.Empty; Author = string.Empty; Version = string.Empty; Rules = new List(); SourceFile = string.Empty; } /// /// Create a Skipper object parsed from an input file /// /// Name of the file to parse public Skipper(string filename) { Rules = new List(); SourceFile = Path.GetFileNameWithoutExtension(filename); XmlReader xtr = filename.GetXmlTextReader(); if (xtr == null) return; bool valid = false; xtr.MoveToContent(); while (!xtr.EOF) { if (xtr.NodeType != XmlNodeType.Element) xtr.Read(); switch (xtr.Name.ToLowerInvariant()) { case "detector": valid = true; xtr.Read(); break; case "name": Name = xtr.ReadElementContentAsString(); break; case "author": Author = xtr.ReadElementContentAsString(); break; case "version": Version = xtr.ReadElementContentAsString(); break; case "rule": // Get the information from the rule first SkipperRule rule = new SkipperRule { StartOffset = null, EndOffset = null, Operation = HeaderSkipOperation.None, Tests = new List(), SourceFile = Path.GetFileNameWithoutExtension(filename), }; if (xtr.GetAttribute("start_offset") != null) { string offset = xtr.GetAttribute("start_offset"); if (offset.ToLowerInvariant() == "eof") { rule.StartOffset = null; } else { rule.StartOffset = Convert.ToInt64(offset, 16); } } if (xtr.GetAttribute("end_offset") != null) { string offset = xtr.GetAttribute("end_offset"); if (offset.ToLowerInvariant() == "eof") { rule.EndOffset = null; } else { rule.EndOffset = Convert.ToInt64(offset, 16); } } if (xtr.GetAttribute("operation") != null) { string operation = xtr.GetAttribute("operation"); switch (operation.ToLowerInvariant()) { case "bitswap": rule.Operation = HeaderSkipOperation.Bitswap; break; case "byteswap": rule.Operation = HeaderSkipOperation.Byteswap; break; case "wordswap": rule.Operation = HeaderSkipOperation.Wordswap; break; } } // Now read the individual tests into the Rule XmlReader subreader = xtr.ReadSubtree(); if (subreader != null) { while (!subreader.EOF) { if (subreader.NodeType != XmlNodeType.Element) subreader.Read(); // Get the test type SkipperTest test = new SkipperTest { Offset = 0, Value = new byte[0], Result = true, Mask = new byte[0], Size = 0, Operator = HeaderSkipTestFileOperator.Equal, }; switch (subreader.Name.ToLowerInvariant()) { case "data": test.Type = HeaderSkipTest.Data; break; case "or": test.Type = HeaderSkipTest.Or; break; case "xor": test.Type = HeaderSkipTest.Xor; break; case "and": test.Type = HeaderSkipTest.And; break; case "file": test.Type = HeaderSkipTest.File; break; default: subreader.Read(); break; } // Now populate all the parts that we can if (subreader.GetAttribute("offset") != null) { string offset = subreader.GetAttribute("offset"); if (offset.ToLowerInvariant() == "eof") test.Offset = null; else test.Offset = Convert.ToInt64(offset, 16); } if (subreader.GetAttribute("value") != null) { string value = subreader.GetAttribute("value"); // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array test.Value = new byte[value.Length / 2]; for (int index = 0; index < test.Value.Length; index++) { string byteValue = value.Substring(index * 2, 2); test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } } if (subreader.GetAttribute("result") != null) { string result = subreader.GetAttribute("result"); if (!bool.TryParse(result, out bool resultBool)) resultBool = true; test.Result = resultBool; } if (subreader.GetAttribute("mask") != null) { string mask = subreader.GetAttribute("mask"); // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array test.Mask = new byte[mask.Length / 2]; for (int index = 0; index < test.Mask.Length; index++) { string byteValue = mask.Substring(index * 2, 2); test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } } if (subreader.GetAttribute("size") != null) { string size = subreader.GetAttribute("size"); if (size.ToLowerInvariant() == "po2") test.Size = null; else test.Size = Convert.ToInt64(size, 16); } if (subreader.GetAttribute("operator") != null) { string oper = subreader.GetAttribute("operator"); #if NET_FRAMEWORK switch (oper.ToLowerInvariant()) { case "less": test.Operator = HeaderSkipTestFileOperator.Less; break; case "greater": test.Operator = HeaderSkipTestFileOperator.Greater; break; case "equal": default: test.Operator = HeaderSkipTestFileOperator.Equal; break; } #else test.Operator = oper.ToLowerInvariant() switch { "less" => HeaderSkipTestFileOperator.Less, "greater" => HeaderSkipTestFileOperator.Greater, "equal" => HeaderSkipTestFileOperator.Equal, _ => HeaderSkipTestFileOperator.Equal, }; #endif } // Add the created test to the rule rule.Tests.Add(test); subreader.Read(); } } // Add the created rule to the skipper Rules.Add(rule); xtr.Skip(); break; default: xtr.Read(); break; } } // If we somehow have an invalid file, zero out the fields if (!valid) { Name = null; Author = null; Version = null; Rules = null; SourceFile = null; } } #endregion #region Static Methods /// /// Initialize static fields /// public static void Init() { PopulateSkippers(); } /// /// Populate the entire list of header Skippers /// /// /// http://mamedev.emulab.it/clrmamepro/docs/xmlheaders.txt /// http://www.emulab.it/forum/index.php?topic=127.0 /// private static void PopulateSkippers() { if (List == null) List = new List(); foreach (string skipperFile in Directory.EnumerateFiles(LocalPath, "*", SearchOption.AllDirectories)) { List.Add(new Skipper(Path.GetFullPath(skipperFile))); } } /// /// Detect header skipper compliance and create an output file /// /// Name of the file to be parsed /// Output directory to write the file to, empty means the same directory as the input file /// True if headers should not be stored in the database, false otherwise /// True if the output file was created, false otherwise public static bool DetectTransformStore(string file, string outDir, bool nostore) { // Create the output directory if it doesn't exist DirectoryExtensions.Ensure(outDir, create: true); Globals.Logger.User($"\nGetting skipper information for '{file}'"); // Get the skipper rule that matches the file, if any SkipperRule rule = GetMatchingRule(file, string.Empty); // If we have an empty rule, return false if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None) return false; Globals.Logger.User("File has a valid copier header"); // Get the header bytes from the file first string hstr; try { // Extract the header as a string for the database #if NET_FRAMEWORK using (var fs = FileExtensions.TryOpenRead(file)) { #else using var fs = FileExtensions.TryOpenRead(file); #endif byte[] hbin = new byte[(int)rule.StartOffset]; fs.Read(hbin, 0, (int)rule.StartOffset); hstr = Utilities.ByteArrayToString(hbin); #if NET_FRAMEWORK } #endif } catch { return false; } // Apply the rule to the file string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file))); rule.TransformFile(file, newfile); // If the output file doesn't exist, return false if (!File.Exists(newfile)) return false; // Now add the information to the database if it's not already there if (!nostore) { BaseFile baseFile = FileExtensions.GetInfo(newfile, chdsAsFiles: true); DatabaseTools.AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile); } return true; } /// /// Detect and replace header(s) to the given file /// /// Name of the file to be parsed /// Output directory to write the file to, empty means the same directory as the input file /// True if a header was found and appended, false otherwise public static bool RestoreHeader(string file, string outDir) { // Create the output directory if it doesn't exist if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir)) Directory.CreateDirectory(outDir); // First, get the SHA-1 hash of the file BaseFile baseFile = FileExtensions.GetInfo(file, chdsAsFiles: true); // Retrieve a list of all related headers from the database List headers = DatabaseTools.RetrieveHeadersFromDatabase(Utilities.ByteArrayToString(baseFile.SHA1)); // If we have nothing retrieved, we return false if (headers.Count == 0) return false; // Now loop through and create the reheadered files, if possible for (int i = 0; i < headers.Count; i++) { string outputFile = (string.IsNullOrWhiteSpace(outDir) ? $"{Path.GetFullPath(file)}.new" : Path.Combine(outDir, Path.GetFileName(file))) + i; Globals.Logger.User($"Creating reheadered file: {outputFile}"); FileExtensions.AppendBytes(file, outputFile, Utilities.StringToByteArray(headers[i]), null); Globals.Logger.User("Reheadered file created!"); } return true; } /// /// Get the SkipperRule associated with a given file /// /// Name of the file to be checked /// Name of the skipper to be used, blank to find a matching skipper /// Logger object for file and console output /// The SkipperRule that matched the file public static SkipperRule GetMatchingRule(string input, string skipperName) { // If the file doesn't exist, return a blank skipper rule if (!File.Exists(input)) { Globals.Logger.Error($"The file '{input}' does not exist so it cannot be tested"); return new SkipperRule(); } return GetMatchingRule(FileExtensions.TryOpenRead(input), skipperName); } /// /// Get the SkipperRule associated with a given stream /// /// Name of the file to be checked /// Name of the skipper to be used, blank to find a matching skipper /// Logger object for file and console output /// True if the underlying stream should be kept open, false otherwise /// The SkipperRule that matched the file public static SkipperRule GetMatchingRule(Stream input, string skipperName, bool keepOpen = false) { SkipperRule skipperRule = new SkipperRule(); // If we have a null skipper name, we return since we're not matching skippers if (skipperName == null) return skipperRule; // Loop through and find a Skipper that has the right name Globals.Logger.Verbose("Beginning search for matching header skip rules"); List tempList = new List(); tempList.AddRange(List); foreach (Skipper skipper in tempList) { // If we're searching for the skipper OR we have a match to an inputted one if (string.IsNullOrWhiteSpace(skipperName) || (!string.IsNullOrWhiteSpace(skipper.Name) && skipperName.ToLowerInvariant() == skipper.Name.ToLowerInvariant()) || (!string.IsNullOrWhiteSpace(skipper.Name) && skipperName.ToLowerInvariant() == skipper.SourceFile.ToLowerInvariant())) { // Loop through the rules until one is found that works BinaryReader br = new BinaryReader(input); foreach (SkipperRule rule in skipper.Rules) { // Always reset the stream back to the original place input.Seek(0, SeekOrigin.Begin); // For each rule, make sure it passes each test bool success = true; foreach (SkipperTest test in rule.Tests) { bool result = true; switch (test.Type) { case HeaderSkipTest.Data: // First seek to the correct position if (test.Offset == null) input.Seek(0, SeekOrigin.End); else if (test.Offset > 0 && test.Offset <= input.Length) input.Seek((long)test.Offset, SeekOrigin.Begin); else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= input.Length) input.Seek((long)test.Offset, SeekOrigin.End); // Then read and compare bytewise result = true; for (int i = 0; i < test.Value.Length; i++) { try { if (br.ReadByte() != test.Value[i]) { result = false; break; } } catch { result = false; break; } } // Return if the expected and actual results match success &= (result == test.Result); break; case HeaderSkipTest.Or: case HeaderSkipTest.Xor: case HeaderSkipTest.And: // First seek to the correct position if (test.Offset == null) input.Seek(0, SeekOrigin.End); else if (test.Offset > 0 && test.Offset <= input.Length) input.Seek((long)test.Offset, SeekOrigin.Begin); else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= input.Length) input.Seek((long)test.Offset, SeekOrigin.End); result = true; try { // Then apply the mask if it exists byte[] read = br.ReadBytes(test.Mask.Length); byte[] masked = new byte[test.Mask.Length]; for (int i = 0; i < read.Length; i++) { masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] : (test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i]) ); } // Finally, compare it against the value for (int i = 0; i < test.Value.Length; i++) { if (masked[i] != test.Value[i]) { result = false; break; } } } catch { result = false; } // Return if the expected and actual results match success &= (result == test.Result); break; case HeaderSkipTest.File: // First get the file size from stream long size = input.Length; // If we have a null size, check that the size is a power of 2 result = true; if (test.Size == null) { // http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2 result = (((ulong)size & ((ulong)size - 1)) == 0); } else if (test.Operator == HeaderSkipTestFileOperator.Less) { result = (size < test.Size); } else if (test.Operator == HeaderSkipTestFileOperator.Greater) { result = (size > test.Size); } else if (test.Operator == HeaderSkipTestFileOperator.Equal) { result = (size == test.Size); } // Return if the expected and actual results match success &= (result == test.Result); break; } } // If we still have a success, then return this rule if (success) { // If we're not keeping the stream open, dispose of the binary reader if (!keepOpen) input.Dispose(); Globals.Logger.User(" Matching rule found!"); return rule; } } } } // If we're not keeping the stream open, dispose of the binary reader if (!keepOpen) input.Dispose(); // If we have a blank rule, inform the user if (skipperRule.Tests == null) Globals.Logger.Verbose("No matching rule found!"); return skipperRule; } #endregion } }