diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index e6d259f1..9324f016 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -1945,7 +1945,7 @@ namespace SabreTools.Library.DatFiles } // Preload the Skipper list - Skipper.Init(); + Transform.Init(); #endregion @@ -2129,7 +2129,7 @@ namespace SabreTools.Library.DatFiles } // Preload the Skipper list - Skipper.Init(); + Transform.Init(); #endregion @@ -2495,7 +2495,7 @@ namespace SabreTools.Library.DatFiles return false; // Check to see if we have a matching header first - SkipperRule rule = Skipper.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(headerToCheckAgainst)); + SkipperRule rule = Transform.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(headerToCheckAgainst)); // If there's a match, create the new file to write if (rule.Tests != null && rule.Tests.Count != 0) diff --git a/SabreTools.Library/DatItems/DatItem.cs b/SabreTools.Library/DatItems/DatItem.cs index cd343983..820af4cf 100644 --- a/SabreTools.Library/DatItems/DatItem.cs +++ b/SabreTools.Library/DatItems/DatItem.cs @@ -17,6 +17,7 @@ namespace SabreTools.Library.DatItems /// public abstract class DatItem : IEquatable, IComparable, ICloneable { + // TODO: Can internal fields be mapped to Field in a more reasonable way? #region Protected instance variables [JsonIgnore] diff --git a/SabreTools.Library/Skippers/SkipperFile.cs b/SabreTools.Library/Skippers/SkipperFile.cs new file mode 100644 index 00000000..a23adb54 --- /dev/null +++ b/SabreTools.Library/Skippers/SkipperFile.cs @@ -0,0 +1,344 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Xml; + +using SabreTools.Library.Data; +using SabreTools.Library.Tools; + +namespace SabreTools.Library.Skippers +{ + public class SkipperFile + { + #region Fields + + /// + /// Skipper name + /// + public string Name { get; set; } = string.Empty; + + /// + /// Author names + /// + public string Author { get; set; } = string.Empty; + + /// + /// File version + /// + public string Version { get; set; } = string.Empty; + + /// + /// Set of all rules in the skipper + /// + public List Rules { get; set; } = new List(); + + /// + /// Filename the skipper lives in + /// + public string SourceFile { get; set; } = string.Empty; + + #endregion + + #region Constructors + + /// + /// Create an empty SkipperFile object + /// + public SkipperFile() { } + + /// + /// Create a SkipperFile object parsed from an input file + /// + /// Name of the file to parse + public SkipperFile(string filename) + { + Rules = new List(); + SourceFile = Path.GetFileNameWithoutExtension(filename); + + XmlReader xtr = filename.GetXmlTextReader(); + bool valid = Parse(xtr); + + // If we somehow have an invalid file, zero out the fields + if (!valid) + { + Name = null; + Author = null; + Version = null; + Rules = null; + SourceFile = null; + } + } + + #endregion + + #region Parsing Helpers + + /// + /// Parse an XML document in as a SkipperFile + /// + /// XmlReader representing the document + private bool Parse(XmlReader xtr) + { + if (xtr == null) + return false; + + try + { + bool valid = false; + xtr.MoveToContent(); + while (!xtr.EOF) + { + if (xtr.NodeType != XmlNodeType.Element) + xtr.Read(); + + switch (xtr.Name.ToLowerInvariant()) + { + case "detector": + valid = true; + xtr.Read(); + break; + + case "name": + Name = xtr.ReadElementContentAsString(); + break; + + case "author": + Author = xtr.ReadElementContentAsString(); + break; + + case "version": + Version = xtr.ReadElementContentAsString(); + break; + + case "rule": + SkipperRule rule = ParseRule(xtr); + if (rule != null) + Rules.Add(rule); + + xtr.Skip(); + break; + default: + xtr.Read(); + break; + } + } + + return valid; + } + catch + { + return false; + } + } + + /// + /// Parse an XML document in as a SkipperRule + /// + /// XmlReader representing the document + private SkipperRule ParseRule(XmlReader xtr) + { + if (xtr == null) + return null; + + try + { + // Get the information from the rule first + SkipperRule rule = new SkipperRule + { + StartOffset = null, + EndOffset = null, + Operation = HeaderSkipOperation.None, + Tests = new List(), + SourceFile = this.SourceFile, + }; + + if (xtr.GetAttribute("start_offset") != null) + { + string offset = xtr.GetAttribute("start_offset"); + if (offset.ToLowerInvariant() == "eof") + { + rule.StartOffset = null; + } + else + { + rule.StartOffset = Convert.ToInt64(offset, 16); + } + } + + if (xtr.GetAttribute("end_offset") != null) + { + string offset = xtr.GetAttribute("end_offset"); + if (offset.ToLowerInvariant() == "eof") + { + rule.EndOffset = null; + } + else + { + rule.EndOffset = Convert.ToInt64(offset, 16); + } + } + + if (xtr.GetAttribute("operation") != null) + { + string operation = xtr.GetAttribute("operation"); + switch (operation.ToLowerInvariant()) + { + case "bitswap": + rule.Operation = HeaderSkipOperation.Bitswap; + break; + case "byteswap": + rule.Operation = HeaderSkipOperation.Byteswap; + break; + case "wordswap": + rule.Operation = HeaderSkipOperation.Wordswap; + break; + } + } + + // Now read the individual tests into the Rule + XmlReader subreader = xtr.ReadSubtree(); + + if (subreader != null) + { + while (!subreader.EOF) + { + if (subreader.NodeType != XmlNodeType.Element) + subreader.Read(); + + // Get the test type + SkipperTest test = new SkipperTest + { + Offset = 0, + Value = new byte[0], + Result = true, + Mask = new byte[0], + Size = 0, + Operator = HeaderSkipTestFileOperator.Equal, + }; + switch (subreader.Name.ToLowerInvariant()) + { + case "data": + test.Type = HeaderSkipTest.Data; + break; + + case "or": + test.Type = HeaderSkipTest.Or; + break; + + case "xor": + test.Type = HeaderSkipTest.Xor; + break; + + case "and": + test.Type = HeaderSkipTest.And; + break; + + case "file": + test.Type = HeaderSkipTest.File; + break; + + default: + subreader.Read(); + break; + } + + // Now populate all the parts that we can + if (subreader.GetAttribute("offset") != null) + { + string offset = subreader.GetAttribute("offset"); + if (offset.ToLowerInvariant() == "eof") + test.Offset = null; + else + test.Offset = Convert.ToInt64(offset, 16); + } + + if (subreader.GetAttribute("value") != null) + { + string value = subreader.GetAttribute("value"); + + // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array + test.Value = new byte[value.Length / 2]; + for (int index = 0; index < test.Value.Length; index++) + { + string byteValue = value.Substring(index * 2, 2); + test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + } + + if (subreader.GetAttribute("result") != null) + { + string result = subreader.GetAttribute("result"); + if (!bool.TryParse(result, out bool resultBool)) + resultBool = true; + + test.Result = resultBool; + } + + if (subreader.GetAttribute("mask") != null) + { + string mask = subreader.GetAttribute("mask"); + + // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array + test.Mask = new byte[mask.Length / 2]; + for (int index = 0; index < test.Mask.Length; index++) + { + string byteValue = mask.Substring(index * 2, 2); + test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + } + } + + if (subreader.GetAttribute("size") != null) + { + string size = subreader.GetAttribute("size"); + if (size.ToLowerInvariant() == "po2") + test.Size = null; + else + test.Size = Convert.ToInt64(size, 16); + } + + if (subreader.GetAttribute("operator") != null) + { + string oper = subreader.GetAttribute("operator"); +#if NET_FRAMEWORK + switch (oper.ToLowerInvariant()) + { + case "less": + test.Operator = HeaderSkipTestFileOperator.Less; + break; + case "greater": + test.Operator = HeaderSkipTestFileOperator.Greater; + break; + case "equal": + default: + test.Operator = HeaderSkipTestFileOperator.Equal; + break; + } +#else + test.Operator = oper.ToLowerInvariant() switch + { + "less" => HeaderSkipTestFileOperator.Less, + "greater" => HeaderSkipTestFileOperator.Greater, + "equal" => HeaderSkipTestFileOperator.Equal, + _ => HeaderSkipTestFileOperator.Equal, + }; +#endif + } + + // Add the created test to the rule + rule.Tests.Add(test); + subreader.Read(); + } + } + + return rule; + } + catch + { + return null; + } + } + + #endregion + } +} diff --git a/SabreTools.Library/Skippers/SkipperRule.cs b/SabreTools.Library/Skippers/SkipperRule.cs index 5187d15f..936b48f0 100644 --- a/SabreTools.Library/Skippers/SkipperRule.cs +++ b/SabreTools.Library/Skippers/SkipperRule.cs @@ -201,45 +201,4 @@ namespace SabreTools.Library.Skippers return success; } } - - /// - /// Intermediate class for storing Skipper Test information - /// - public struct SkipperTest - { - /// - /// Type of test to be run - /// - public HeaderSkipTest Type { get; set; } - - /// - /// File offset to run the test - /// - public long? Offset { get; set; } // null is EOF - - /// - /// Static value to be checked at the offset - /// - public byte[] Value { get; set; } - - /// - /// Determines whether a pass or failure is expected - /// - public bool Result { get; set; } - - /// - /// Byte mask to be applied to the tested bytes - /// - public byte[] Mask { get; set; } - - /// - /// Expected size of the input byte array, used with the Operator - /// - public long? Size { get; set; } // null is PO2, "power of 2" filesize - - /// - /// Expected range value for the input byte array size, used with Size - /// - public HeaderSkipTestFileOperator Operator { get; set; } - } } diff --git a/SabreTools.Library/Skippers/SkipperTest.cs b/SabreTools.Library/Skippers/SkipperTest.cs new file mode 100644 index 00000000..d3f3f98b --- /dev/null +++ b/SabreTools.Library/Skippers/SkipperTest.cs @@ -0,0 +1,45 @@ +using SabreTools.Library.Data; + +namespace SabreTools.Library.Skippers +{ + /// + /// Intermediate class for storing Skipper Test information + /// + public struct SkipperTest + { + /// + /// Type of test to be run + /// + public HeaderSkipTest Type { get; set; } + + /// + /// File offset to run the test + /// + public long? Offset { get; set; } // null is EOF + + /// + /// Static value to be checked at the offset + /// + public byte[] Value { get; set; } + + /// + /// Determines whether a pass or failure is expected + /// + public bool Result { get; set; } + + /// + /// Byte mask to be applied to the tested bytes + /// + public byte[] Mask { get; set; } + + /// + /// Expected size of the input byte array, used with the Operator + /// + public long? Size { get; set; } // null is PO2, "power of 2" filesize + + /// + /// Expected range value for the input byte array size, used with Size + /// + public HeaderSkipTestFileOperator Operator { get; set; } + } +} \ No newline at end of file diff --git a/SabreTools.Library/Skippers/Skipper.cs b/SabreTools.Library/Skippers/Transform.cs similarity index 54% rename from SabreTools.Library/Skippers/Skipper.cs rename to SabreTools.Library/Skippers/Transform.cs index 6ca94556..9021c5ca 100644 --- a/SabreTools.Library/Skippers/Skipper.cs +++ b/SabreTools.Library/Skippers/Transform.cs @@ -1,323 +1,40 @@ -using SabreTools.Library.Data; -using SabreTools.Library.FileTypes; -using SabreTools.Library.Tools; using System; using System.Collections.Generic; -using System.Globalization; using System.IO; -using System.Xml; + +using SabreTools.Library.Data; +using SabreTools.Library.FileTypes; +using SabreTools.Library.Tools; namespace SabreTools.Library.Skippers { - public class Skipper + /// + /// Class for wrapping general file transformations + /// + /// + /// Skippers, in general, are distributed as XML files by some projects + /// in order to denote a way of transforming a file so that it will match + /// the hashes included in their DATs. Each skipper file can contain multiple + /// skipper rules, each of which denote a type of header/transformation. In + /// turn, each of those rules can contain multiple tests that denote that + /// a file should be processed using that rule. Transformations can include + /// simply skipping over a portion of the file all the way to byteswapping + /// the entire file. For the purposes of this library, Skippers also denote + /// a way of changing files directly in order to produce a file whose external + /// hash would match those same DATs. + /// + public static class Transform { - #region Fields - /// - /// Skipper name + /// Header skippers represented by a list of skipper objects /// - public string Name { get; set; } - - /// - /// Author names - /// - public string Author { get; set; } - - /// - /// File version - /// - public string Version { get; set; } - - /// - /// Set of all rules in the skipper - /// - public List Rules { get; set; } - - /// - /// Filename the skipper lives in - /// - public string SourceFile { get; set; } + private static List List; /// /// Local paths /// public static string LocalPath = Path.Combine(Globals.ExeDir, "Skippers") + Path.DirectorySeparatorChar; - /// - /// Header skippers represented by a list of skipper objects - /// - private static List List; - - #endregion - - #region Constructors - - /// - /// Create an empty Skipper object - /// - public Skipper() - { - Name = string.Empty; - Author = string.Empty; - Version = string.Empty; - Rules = new List(); - SourceFile = string.Empty; - } - - /// - /// Create a Skipper object parsed from an input file - /// - /// Name of the file to parse - public Skipper(string filename) - { - Rules = new List(); - SourceFile = Path.GetFileNameWithoutExtension(filename); - - XmlReader xtr = filename.GetXmlTextReader(); - - if (xtr == null) - return; - - bool valid = false; - xtr.MoveToContent(); - while (!xtr.EOF) - { - if (xtr.NodeType != XmlNodeType.Element) - xtr.Read(); - - switch (xtr.Name.ToLowerInvariant()) - { - case "detector": - valid = true; - xtr.Read(); - break; - - case "name": - Name = xtr.ReadElementContentAsString(); - break; - - case "author": - Author = xtr.ReadElementContentAsString(); - break; - - case "version": - Version = xtr.ReadElementContentAsString(); - break; - - case "rule": - // Get the information from the rule first - SkipperRule rule = new SkipperRule - { - StartOffset = null, - EndOffset = null, - Operation = HeaderSkipOperation.None, - Tests = new List(), - SourceFile = Path.GetFileNameWithoutExtension(filename), - }; - - if (xtr.GetAttribute("start_offset") != null) - { - string offset = xtr.GetAttribute("start_offset"); - if (offset.ToLowerInvariant() == "eof") - { - rule.StartOffset = null; - } - else - { - rule.StartOffset = Convert.ToInt64(offset, 16); - } - } - - if (xtr.GetAttribute("end_offset") != null) - { - string offset = xtr.GetAttribute("end_offset"); - if (offset.ToLowerInvariant() == "eof") - { - rule.EndOffset = null; - } - else - { - rule.EndOffset = Convert.ToInt64(offset, 16); - } - } - - if (xtr.GetAttribute("operation") != null) - { - string operation = xtr.GetAttribute("operation"); - switch (operation.ToLowerInvariant()) - { - case "bitswap": - rule.Operation = HeaderSkipOperation.Bitswap; - break; - case "byteswap": - rule.Operation = HeaderSkipOperation.Byteswap; - break; - case "wordswap": - rule.Operation = HeaderSkipOperation.Wordswap; - break; - } - } - - // Now read the individual tests into the Rule - XmlReader subreader = xtr.ReadSubtree(); - - if (subreader != null) - { - while (!subreader.EOF) - { - if (subreader.NodeType != XmlNodeType.Element) - subreader.Read(); - - // Get the test type - SkipperTest test = new SkipperTest - { - Offset = 0, - Value = new byte[0], - Result = true, - Mask = new byte[0], - Size = 0, - Operator = HeaderSkipTestFileOperator.Equal, - }; - switch (subreader.Name.ToLowerInvariant()) - { - case "data": - test.Type = HeaderSkipTest.Data; - break; - - case "or": - test.Type = HeaderSkipTest.Or; - break; - - case "xor": - test.Type = HeaderSkipTest.Xor; - break; - - case "and": - test.Type = HeaderSkipTest.And; - break; - - case "file": - test.Type = HeaderSkipTest.File; - break; - - default: - subreader.Read(); - break; - } - - // Now populate all the parts that we can - if (subreader.GetAttribute("offset") != null) - { - string offset = subreader.GetAttribute("offset"); - if (offset.ToLowerInvariant() == "eof") - test.Offset = null; - else - test.Offset = Convert.ToInt64(offset, 16); - } - - if (subreader.GetAttribute("value") != null) - { - string value = subreader.GetAttribute("value"); - - // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array - test.Value = new byte[value.Length / 2]; - for (int index = 0; index < test.Value.Length; index++) - { - string byteValue = value.Substring(index * 2, 2); - test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); - } - } - - if (subreader.GetAttribute("result") != null) - { - string result = subreader.GetAttribute("result"); - if (!bool.TryParse(result, out bool resultBool)) - resultBool = true; - - test.Result = resultBool; - } - - if (subreader.GetAttribute("mask") != null) - { - string mask = subreader.GetAttribute("mask"); - - // http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array - test.Mask = new byte[mask.Length / 2]; - for (int index = 0; index < test.Mask.Length; index++) - { - string byteValue = mask.Substring(index * 2, 2); - test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture); - } - } - - if (subreader.GetAttribute("size") != null) - { - string size = subreader.GetAttribute("size"); - if (size.ToLowerInvariant() == "po2") - test.Size = null; - else - test.Size = Convert.ToInt64(size, 16); - } - - if (subreader.GetAttribute("operator") != null) - { - string oper = subreader.GetAttribute("operator"); -#if NET_FRAMEWORK - switch (oper.ToLowerInvariant()) - { - case "less": - test.Operator = HeaderSkipTestFileOperator.Less; - break; - case "greater": - test.Operator = HeaderSkipTestFileOperator.Greater; - break; - case "equal": - default: - test.Operator = HeaderSkipTestFileOperator.Equal; - break; - } -#else - test.Operator = oper.ToLowerInvariant() switch - { - "less" => HeaderSkipTestFileOperator.Less, - "greater" => HeaderSkipTestFileOperator.Greater, - "equal" => HeaderSkipTestFileOperator.Equal, - _ => HeaderSkipTestFileOperator.Equal, - }; -#endif - } - - // Add the created test to the rule - rule.Tests.Add(test); - subreader.Read(); - } - } - - // Add the created rule to the skipper - Rules.Add(rule); - xtr.Skip(); - break; - default: - xtr.Read(); - break; - } - } - - // If we somehow have an invalid file, zero out the fields - if (!valid) - { - Name = null; - Author = null; - Version = null; - Rules = null; - SourceFile = null; - } - } - - #endregion - - #region Static Methods - /// /// Initialize static fields /// @@ -336,11 +53,11 @@ namespace SabreTools.Library.Skippers private static void PopulateSkippers() { if (List == null) - List = new List(); + List = new List(); foreach (string skipperFile in Directory.EnumerateFiles(LocalPath, "*", SearchOption.AllDirectories)) { - List.Add(new Skipper(Path.GetFullPath(skipperFile))); + List.Add(new SkipperFile(Path.GetFullPath(skipperFile))); } } @@ -479,10 +196,10 @@ namespace SabreTools.Library.Skippers // Loop through and find a Skipper that has the right name Globals.Logger.Verbose("Beginning search for matching header skip rules"); - List tempList = new List(); + List tempList = new List(); tempList.AddRange(List); - foreach (Skipper skipper in tempList) + foreach (SkipperFile skipper in tempList) { // If we're searching for the skipper OR we have a match to an inputted one if (string.IsNullOrWhiteSpace(skipperName) @@ -633,7 +350,5 @@ namespace SabreTools.Library.Skippers return skipperRule; } - - #endregion } -} +} \ No newline at end of file diff --git a/SabreTools.Library/Tools/FileExtensions.cs b/SabreTools.Library/Tools/FileExtensions.cs index 9ee12cf9..c31da7de 100644 --- a/SabreTools.Library/Tools/FileExtensions.cs +++ b/SabreTools.Library/Tools/FileExtensions.cs @@ -347,7 +347,7 @@ namespace SabreTools.Library.Tools BaseFile baseFile; if (header != null) { - SkipperRule rule = Skipper.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header)); + SkipperRule rule = Transform.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header)); // If there's a match, get the new information from the stream if (rule.Tests != null && rule.Tests.Count != 0) diff --git a/SabreTools/SabreTools.Help.cs b/SabreTools/SabreTools.Help.cs index 58733ea8..c1f8746d 100644 --- a/SabreTools/SabreTools.Help.cs +++ b/SabreTools/SabreTools.Help.cs @@ -2982,7 +2982,7 @@ The following systems have headers that this program can work with: List files = DirectoryExtensions.GetFilesOnly(Inputs); foreach (ParentablePath file in files) { - Skipper.DetectTransformStore(file.CurrentPath, outDir, nostore); + Transform.DetectTransformStore(file.CurrentPath, outDir, nostore); } } } @@ -3054,7 +3054,7 @@ The following systems have headers that this program can work with: List files = DirectoryExtensions.GetFilesOnly(Inputs); foreach (ParentablePath file in files) { - Skipper.RestoreHeader(file.CurrentPath, outDir); + Transform.RestoreHeader(file.CurrentPath, outDir); } } }