Clean up Skipper structure

This commit is contained in:
Matt Nadareski
2020-07-30 21:07:25 -07:00
parent 88d0d043ef
commit a4b2a4ff17
8 changed files with 423 additions and 359 deletions

View File

@@ -1945,7 +1945,7 @@ namespace SabreTools.Library.DatFiles
}
// Preload the Skipper list
Skipper.Init();
Transform.Init();
#endregion
@@ -2129,7 +2129,7 @@ namespace SabreTools.Library.DatFiles
}
// Preload the Skipper list
Skipper.Init();
Transform.Init();
#endregion
@@ -2495,7 +2495,7 @@ namespace SabreTools.Library.DatFiles
return false;
// Check to see if we have a matching header first
SkipperRule rule = Skipper.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(headerToCheckAgainst));
SkipperRule rule = Transform.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(headerToCheckAgainst));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)

View File

@@ -17,6 +17,7 @@ namespace SabreTools.Library.DatItems
/// </summary>
public abstract class DatItem : IEquatable<DatItem>, IComparable<DatItem>, ICloneable
{
// TODO: Can internal fields be mapped to Field in a more reasonable way?
#region Protected instance variables
[JsonIgnore]

View File

@@ -0,0 +1,344 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Xml;
using SabreTools.Library.Data;
using SabreTools.Library.Tools;
namespace SabreTools.Library.Skippers
{
public class SkipperFile
{
#region Fields
/// <summary>
/// Skipper name
/// </summary>
public string Name { get; set; } = string.Empty;
/// <summary>
/// Author names
/// </summary>
public string Author { get; set; } = string.Empty;
/// <summary>
/// File version
/// </summary>
public string Version { get; set; } = string.Empty;
/// <summary>
/// Set of all rules in the skipper
/// </summary>
public List<SkipperRule> Rules { get; set; } = new List<SkipperRule>();
/// <summary>
/// Filename the skipper lives in
/// </summary>
public string SourceFile { get; set; } = string.Empty;
#endregion
#region Constructors
/// <summary>
/// Create an empty SkipperFile object
/// </summary>
public SkipperFile() { }
/// <summary>
/// Create a SkipperFile object parsed from an input file
/// </summary>
/// <param name="filename">Name of the file to parse</param>
public SkipperFile(string filename)
{
Rules = new List<SkipperRule>();
SourceFile = Path.GetFileNameWithoutExtension(filename);
XmlReader xtr = filename.GetXmlTextReader();
bool valid = Parse(xtr);
// If we somehow have an invalid file, zero out the fields
if (!valid)
{
Name = null;
Author = null;
Version = null;
Rules = null;
SourceFile = null;
}
}
#endregion
#region Parsing Helpers
/// <summary>
/// Parse an XML document in as a SkipperFile
/// </summary>
/// <param name="xtr">XmlReader representing the document</param>
private bool Parse(XmlReader xtr)
{
if (xtr == null)
return false;
try
{
bool valid = false;
xtr.MoveToContent();
while (!xtr.EOF)
{
if (xtr.NodeType != XmlNodeType.Element)
xtr.Read();
switch (xtr.Name.ToLowerInvariant())
{
case "detector":
valid = true;
xtr.Read();
break;
case "name":
Name = xtr.ReadElementContentAsString();
break;
case "author":
Author = xtr.ReadElementContentAsString();
break;
case "version":
Version = xtr.ReadElementContentAsString();
break;
case "rule":
SkipperRule rule = ParseRule(xtr);
if (rule != null)
Rules.Add(rule);
xtr.Skip();
break;
default:
xtr.Read();
break;
}
}
return valid;
}
catch
{
return false;
}
}
/// <summary>
/// Parse an XML document in as a SkipperRule
/// </summary>
/// <param name="xtr">XmlReader representing the document</param>
private SkipperRule ParseRule(XmlReader xtr)
{
if (xtr == null)
return null;
try
{
// Get the information from the rule first
SkipperRule rule = new SkipperRule
{
StartOffset = null,
EndOffset = null,
Operation = HeaderSkipOperation.None,
Tests = new List<SkipperTest>(),
SourceFile = this.SourceFile,
};
if (xtr.GetAttribute("start_offset") != null)
{
string offset = xtr.GetAttribute("start_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.StartOffset = null;
}
else
{
rule.StartOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("end_offset") != null)
{
string offset = xtr.GetAttribute("end_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.EndOffset = null;
}
else
{
rule.EndOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("operation") != null)
{
string operation = xtr.GetAttribute("operation");
switch (operation.ToLowerInvariant())
{
case "bitswap":
rule.Operation = HeaderSkipOperation.Bitswap;
break;
case "byteswap":
rule.Operation = HeaderSkipOperation.Byteswap;
break;
case "wordswap":
rule.Operation = HeaderSkipOperation.Wordswap;
break;
}
}
// Now read the individual tests into the Rule
XmlReader subreader = xtr.ReadSubtree();
if (subreader != null)
{
while (!subreader.EOF)
{
if (subreader.NodeType != XmlNodeType.Element)
subreader.Read();
// Get the test type
SkipperTest test = new SkipperTest
{
Offset = 0,
Value = new byte[0],
Result = true,
Mask = new byte[0],
Size = 0,
Operator = HeaderSkipTestFileOperator.Equal,
};
switch (subreader.Name.ToLowerInvariant())
{
case "data":
test.Type = HeaderSkipTest.Data;
break;
case "or":
test.Type = HeaderSkipTest.Or;
break;
case "xor":
test.Type = HeaderSkipTest.Xor;
break;
case "and":
test.Type = HeaderSkipTest.And;
break;
case "file":
test.Type = HeaderSkipTest.File;
break;
default:
subreader.Read();
break;
}
// Now populate all the parts that we can
if (subreader.GetAttribute("offset") != null)
{
string offset = subreader.GetAttribute("offset");
if (offset.ToLowerInvariant() == "eof")
test.Offset = null;
else
test.Offset = Convert.ToInt64(offset, 16);
}
if (subreader.GetAttribute("value") != null)
{
string value = subreader.GetAttribute("value");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Value = new byte[value.Length / 2];
for (int index = 0; index < test.Value.Length; index++)
{
string byteValue = value.Substring(index * 2, 2);
test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("result") != null)
{
string result = subreader.GetAttribute("result");
if (!bool.TryParse(result, out bool resultBool))
resultBool = true;
test.Result = resultBool;
}
if (subreader.GetAttribute("mask") != null)
{
string mask = subreader.GetAttribute("mask");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Mask = new byte[mask.Length / 2];
for (int index = 0; index < test.Mask.Length; index++)
{
string byteValue = mask.Substring(index * 2, 2);
test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("size") != null)
{
string size = subreader.GetAttribute("size");
if (size.ToLowerInvariant() == "po2")
test.Size = null;
else
test.Size = Convert.ToInt64(size, 16);
}
if (subreader.GetAttribute("operator") != null)
{
string oper = subreader.GetAttribute("operator");
#if NET_FRAMEWORK
switch (oper.ToLowerInvariant())
{
case "less":
test.Operator = HeaderSkipTestFileOperator.Less;
break;
case "greater":
test.Operator = HeaderSkipTestFileOperator.Greater;
break;
case "equal":
default:
test.Operator = HeaderSkipTestFileOperator.Equal;
break;
}
#else
test.Operator = oper.ToLowerInvariant() switch
{
"less" => HeaderSkipTestFileOperator.Less,
"greater" => HeaderSkipTestFileOperator.Greater,
"equal" => HeaderSkipTestFileOperator.Equal,
_ => HeaderSkipTestFileOperator.Equal,
};
#endif
}
// Add the created test to the rule
rule.Tests.Add(test);
subreader.Read();
}
}
return rule;
}
catch
{
return null;
}
}
#endregion
}
}

View File

@@ -201,45 +201,4 @@ namespace SabreTools.Library.Skippers
return success;
}
}
/// <summary>
/// Intermediate class for storing Skipper Test information
/// </summary>
public struct SkipperTest
{
/// <summary>
/// Type of test to be run
/// </summary>
public HeaderSkipTest Type { get; set; }
/// <summary>
/// File offset to run the test
/// </summary>
public long? Offset { get; set; } // null is EOF
/// <summary>
/// Static value to be checked at the offset
/// </summary>
public byte[] Value { get; set; }
/// <summary>
/// Determines whether a pass or failure is expected
/// </summary>
public bool Result { get; set; }
/// <summary>
/// Byte mask to be applied to the tested bytes
/// </summary>
public byte[] Mask { get; set; }
/// <summary>
/// Expected size of the input byte array, used with the Operator
/// </summary>
public long? Size { get; set; } // null is PO2, "power of 2" filesize
/// <summary>
/// Expected range value for the input byte array size, used with Size
/// </summary>
public HeaderSkipTestFileOperator Operator { get; set; }
}
}

View File

@@ -0,0 +1,45 @@
using SabreTools.Library.Data;
namespace SabreTools.Library.Skippers
{
/// <summary>
/// Intermediate class for storing Skipper Test information
/// </summary>
public struct SkipperTest
{
/// <summary>
/// Type of test to be run
/// </summary>
public HeaderSkipTest Type { get; set; }
/// <summary>
/// File offset to run the test
/// </summary>
public long? Offset { get; set; } // null is EOF
/// <summary>
/// Static value to be checked at the offset
/// </summary>
public byte[] Value { get; set; }
/// <summary>
/// Determines whether a pass or failure is expected
/// </summary>
public bool Result { get; set; }
/// <summary>
/// Byte mask to be applied to the tested bytes
/// </summary>
public byte[] Mask { get; set; }
/// <summary>
/// Expected size of the input byte array, used with the Operator
/// </summary>
public long? Size { get; set; } // null is PO2, "power of 2" filesize
/// <summary>
/// Expected range value for the input byte array size, used with Size
/// </summary>
public HeaderSkipTestFileOperator Operator { get; set; }
}
}

View File

@@ -1,323 +1,40 @@
using SabreTools.Library.Data;
using SabreTools.Library.FileTypes;
using SabreTools.Library.Tools;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Xml;
using SabreTools.Library.Data;
using SabreTools.Library.FileTypes;
using SabreTools.Library.Tools;
namespace SabreTools.Library.Skippers
{
public class Skipper
/// <summary>
/// Class for wrapping general file transformations
/// <summary>
/// <remarks>
/// Skippers, in general, are distributed as XML files by some projects
/// in order to denote a way of transforming a file so that it will match
/// the hashes included in their DATs. Each skipper file can contain multiple
/// skipper rules, each of which denote a type of header/transformation. In
/// turn, each of those rules can contain multiple tests that denote that
/// a file should be processed using that rule. Transformations can include
/// simply skipping over a portion of the file all the way to byteswapping
/// the entire file. For the purposes of this library, Skippers also denote
/// a way of changing files directly in order to produce a file whose external
/// hash would match those same DATs.
/// </remarks>
public static class Transform
{
#region Fields
/// <summary>
/// Skipper name
/// Header skippers represented by a list of skipper objects
/// </summary>
public string Name { get; set; }
/// <summary>
/// Author names
/// </summary>
public string Author { get; set; }
/// <summary>
/// File version
/// </summary>
public string Version { get; set; }
/// <summary>
/// Set of all rules in the skipper
/// </summary>
public List<SkipperRule> Rules { get; set; }
/// <summary>
/// Filename the skipper lives in
/// </summary>
public string SourceFile { get; set; }
private static List<SkipperFile> List;
/// <summary>
/// Local paths
/// </summary>
public static string LocalPath = Path.Combine(Globals.ExeDir, "Skippers") + Path.DirectorySeparatorChar;
/// <summary>
/// Header skippers represented by a list of skipper objects
/// </summary>
private static List<Skipper> List;
#endregion
#region Constructors
/// <summary>
/// Create an empty Skipper object
/// </summary>
public Skipper()
{
Name = string.Empty;
Author = string.Empty;
Version = string.Empty;
Rules = new List<SkipperRule>();
SourceFile = string.Empty;
}
/// <summary>
/// Create a Skipper object parsed from an input file
/// </summary>
/// <param name="filename">Name of the file to parse</param>
public Skipper(string filename)
{
Rules = new List<SkipperRule>();
SourceFile = Path.GetFileNameWithoutExtension(filename);
XmlReader xtr = filename.GetXmlTextReader();
if (xtr == null)
return;
bool valid = false;
xtr.MoveToContent();
while (!xtr.EOF)
{
if (xtr.NodeType != XmlNodeType.Element)
xtr.Read();
switch (xtr.Name.ToLowerInvariant())
{
case "detector":
valid = true;
xtr.Read();
break;
case "name":
Name = xtr.ReadElementContentAsString();
break;
case "author":
Author = xtr.ReadElementContentAsString();
break;
case "version":
Version = xtr.ReadElementContentAsString();
break;
case "rule":
// Get the information from the rule first
SkipperRule rule = new SkipperRule
{
StartOffset = null,
EndOffset = null,
Operation = HeaderSkipOperation.None,
Tests = new List<SkipperTest>(),
SourceFile = Path.GetFileNameWithoutExtension(filename),
};
if (xtr.GetAttribute("start_offset") != null)
{
string offset = xtr.GetAttribute("start_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.StartOffset = null;
}
else
{
rule.StartOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("end_offset") != null)
{
string offset = xtr.GetAttribute("end_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.EndOffset = null;
}
else
{
rule.EndOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("operation") != null)
{
string operation = xtr.GetAttribute("operation");
switch (operation.ToLowerInvariant())
{
case "bitswap":
rule.Operation = HeaderSkipOperation.Bitswap;
break;
case "byteswap":
rule.Operation = HeaderSkipOperation.Byteswap;
break;
case "wordswap":
rule.Operation = HeaderSkipOperation.Wordswap;
break;
}
}
// Now read the individual tests into the Rule
XmlReader subreader = xtr.ReadSubtree();
if (subreader != null)
{
while (!subreader.EOF)
{
if (subreader.NodeType != XmlNodeType.Element)
subreader.Read();
// Get the test type
SkipperTest test = new SkipperTest
{
Offset = 0,
Value = new byte[0],
Result = true,
Mask = new byte[0],
Size = 0,
Operator = HeaderSkipTestFileOperator.Equal,
};
switch (subreader.Name.ToLowerInvariant())
{
case "data":
test.Type = HeaderSkipTest.Data;
break;
case "or":
test.Type = HeaderSkipTest.Or;
break;
case "xor":
test.Type = HeaderSkipTest.Xor;
break;
case "and":
test.Type = HeaderSkipTest.And;
break;
case "file":
test.Type = HeaderSkipTest.File;
break;
default:
subreader.Read();
break;
}
// Now populate all the parts that we can
if (subreader.GetAttribute("offset") != null)
{
string offset = subreader.GetAttribute("offset");
if (offset.ToLowerInvariant() == "eof")
test.Offset = null;
else
test.Offset = Convert.ToInt64(offset, 16);
}
if (subreader.GetAttribute("value") != null)
{
string value = subreader.GetAttribute("value");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Value = new byte[value.Length / 2];
for (int index = 0; index < test.Value.Length; index++)
{
string byteValue = value.Substring(index * 2, 2);
test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("result") != null)
{
string result = subreader.GetAttribute("result");
if (!bool.TryParse(result, out bool resultBool))
resultBool = true;
test.Result = resultBool;
}
if (subreader.GetAttribute("mask") != null)
{
string mask = subreader.GetAttribute("mask");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Mask = new byte[mask.Length / 2];
for (int index = 0; index < test.Mask.Length; index++)
{
string byteValue = mask.Substring(index * 2, 2);
test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("size") != null)
{
string size = subreader.GetAttribute("size");
if (size.ToLowerInvariant() == "po2")
test.Size = null;
else
test.Size = Convert.ToInt64(size, 16);
}
if (subreader.GetAttribute("operator") != null)
{
string oper = subreader.GetAttribute("operator");
#if NET_FRAMEWORK
switch (oper.ToLowerInvariant())
{
case "less":
test.Operator = HeaderSkipTestFileOperator.Less;
break;
case "greater":
test.Operator = HeaderSkipTestFileOperator.Greater;
break;
case "equal":
default:
test.Operator = HeaderSkipTestFileOperator.Equal;
break;
}
#else
test.Operator = oper.ToLowerInvariant() switch
{
"less" => HeaderSkipTestFileOperator.Less,
"greater" => HeaderSkipTestFileOperator.Greater,
"equal" => HeaderSkipTestFileOperator.Equal,
_ => HeaderSkipTestFileOperator.Equal,
};
#endif
}
// Add the created test to the rule
rule.Tests.Add(test);
subreader.Read();
}
}
// Add the created rule to the skipper
Rules.Add(rule);
xtr.Skip();
break;
default:
xtr.Read();
break;
}
}
// If we somehow have an invalid file, zero out the fields
if (!valid)
{
Name = null;
Author = null;
Version = null;
Rules = null;
SourceFile = null;
}
}
#endregion
#region Static Methods
/// <summary>
/// Initialize static fields
/// </summary>
@@ -336,11 +53,11 @@ namespace SabreTools.Library.Skippers
private static void PopulateSkippers()
{
if (List == null)
List = new List<Skipper>();
List = new List<SkipperFile>();
foreach (string skipperFile in Directory.EnumerateFiles(LocalPath, "*", SearchOption.AllDirectories))
{
List.Add(new Skipper(Path.GetFullPath(skipperFile)));
List.Add(new SkipperFile(Path.GetFullPath(skipperFile)));
}
}
@@ -479,10 +196,10 @@ namespace SabreTools.Library.Skippers
// Loop through and find a Skipper that has the right name
Globals.Logger.Verbose("Beginning search for matching header skip rules");
List<Skipper> tempList = new List<Skipper>();
List<SkipperFile> tempList = new List<SkipperFile>();
tempList.AddRange(List);
foreach (Skipper skipper in tempList)
foreach (SkipperFile skipper in tempList)
{
// If we're searching for the skipper OR we have a match to an inputted one
if (string.IsNullOrWhiteSpace(skipperName)
@@ -633,7 +350,5 @@ namespace SabreTools.Library.Skippers
return skipperRule;
}
#endregion
}
}

View File

@@ -347,7 +347,7 @@ namespace SabreTools.Library.Tools
BaseFile baseFile;
if (header != null)
{
SkipperRule rule = Skipper.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header));
SkipperRule rule = Transform.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header));
// If there's a match, get the new information from the stream
if (rule.Tests != null && rule.Tests.Count != 0)

View File

@@ -2982,7 +2982,7 @@ The following systems have headers that this program can work with:
List<ParentablePath> files = DirectoryExtensions.GetFilesOnly(Inputs);
foreach (ParentablePath file in files)
{
Skipper.DetectTransformStore(file.CurrentPath, outDir, nostore);
Transform.DetectTransformStore(file.CurrentPath, outDir, nostore);
}
}
}
@@ -3054,7 +3054,7 @@ The following systems have headers that this program can work with:
List<ParentablePath> files = DirectoryExtensions.GetFilesOnly(Inputs);
foreach (ParentablePath file in files)
{
Skipper.RestoreHeader(file.CurrentPath, outDir);
Transform.RestoreHeader(file.CurrentPath, outDir);
}
}
}