claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Skippers] Make header skippers more complete

Browse Source
This commit is contained in:
Matt Nadareski
2016-06-17 11:02:38 -07:00
parent 8143c80be1
commit ae796b98cc
3 changed files with 250 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Deheader/Headerer.cs
View File

@@ -76,7 +76,7 @@ namespace SabreTools
// If it's a single file, just check it // If it's a single file, just check it
if (File.Exists(file)) if (File.Exists(file))
{ {
DetectRemoveHeader(file); DetectSkipperAndTransform(file);
} }
// If it's a directory, recursively check all // If it's a directory, recursively check all
else if (Directory.Exists(file)) else if (Directory.Exists(file))
@@ -85,7 +85,7 @@ namespace SabreTools
{ {
if (sub != ".." && sub != ".") if (sub != ".." && sub != ".")
{ {
DetectRemoveHeader(sub); DetectSkipperAndTransform(sub);
} }
} }
} }
@@ -123,12 +123,15 @@ namespace SabreTools
} }
/// <summary> /// <summary>
/// Detect and remove header from the given file /// Detect header skipper compliance and create an output file
/// </summary> /// </summary>
/// <param name="file">Name of the file to be parsed</param> /// <param name="file">Name of the file to be parsed</param>
private static void DetectRemoveHeader(string file) /// <returns>True if the output file was created, false otherwise</returns>
private static bool DetectSkipperAndTransform(string file)
{ {
// First get the HeaderType, if any logger.User("\nGetting skipper information for '" + file + "'");
// Then, if the file was headered, store it to the database
int headerSize = -1; int headerSize = -1;
HeaderType type = Skippers.GetFileHeaderType(file, out headerSize, logger); HeaderType type = Skippers.GetFileHeaderType(file, out headerSize, logger);
@@ -150,15 +153,30 @@ namespace SabreTools
} }
} }
// Write out the remaining bytes to new file // Then find an apply the exact rule to the file
logger.User("Creating unheadered file: " + file + ".new"); SkipperRule rule = Skippers.MatchesSkipper(file, "", logger);
Output.RemoveBytesFromFile(file, file + ".new", headerSize, 0);
logger.User("Unheadered file created!"); // If we have an empty rule, return false
if (rule.Tests == null || rule.Tests.Count == 0)
{
return false;
}
// Otherwise, apply the rule ot the file
Skippers.TransformFile(file, file + ".new", rule, logger);
// If the output file doesn't exist, return false
if (!File.Exists(file + ".new"))
{
return false;
}
// Now add the information to the database if it's not already there // Now add the information to the database if it's not already there
Rom rom = RomTools.GetSingleFileInfo(file + ".new"); Rom rom = RomTools.GetSingleFileInfo(file + ".new");
AddHeaderToDatabase(hstr, rom.SHA1, type); AddHeaderToDatabase(hstr, rom.SHA1, type);
} }
return true;
} }
/// <summary> /// <summary>

428
SabreTools.Helper/Skippers.cs
View File

@@ -10,7 +10,7 @@ namespace SabreTools.Helper
public class Skippers public class Skippers
{ {
// Local paths // Local paths
private const string _skippersPath = "Skippers"; public const string Path = "Skippers";
// Header skippers represented by a list of skipper objects // Header skippers represented by a list of skipper objects
private static List<Skipper> _list; private static List<Skipper> _list;
@@ -56,9 +56,9 @@ namespace SabreTools.Helper
_list = new List<Skipper>(); _list = new List<Skipper>();
} }
foreach (string skipperFile in Directory.EnumerateFiles(_skippersPath, "*", SearchOption.AllDirectories)) foreach (string skipperFile in Directory.EnumerateFiles(Path, "*", SearchOption.AllDirectories))
{ {
_list.Add(PopulateSkippersHelper(Path.GetFullPath(skipperFile))); _list.Add(PopulateSkippersHelper(System.IO.Path.GetFullPath(skipperFile)));
} }
} }
@@ -69,7 +69,10 @@ namespace SabreTools.Helper
/// <returns>The Skipper object associated with the file</returns> /// <returns>The Skipper object associated with the file</returns>
private static Skipper PopulateSkippersHelper(string filename) private static Skipper PopulateSkippersHelper(string filename)
{ {
Skipper skipper = new Skipper(); Skipper skipper = new Skipper
{
Rules = new List<SkipperRule>(),
};
if (!File.Exists(filename)) if (!File.Exists(filename))
{ {
@@ -98,6 +101,7 @@ namespace SabreTools.Helper
{ {
case "detector": case "detector":
valid = true; valid = true;
xtr.Read();
break; break;
case "name": case "name":
skipper.Name = xtr.ReadElementContentAsString(); skipper.Name = xtr.ReadElementContentAsString();
@@ -127,9 +131,7 @@ namespace SabreTools.Helper
} }
else else
{ {
long temp = 0; rule.StartOffset = Convert.ToInt64(offset, 16);
Int64.TryParse(offset, out temp);
rule.StartOffset = temp;
} }
} }
if (xtr.GetAttribute("end_offset") != null) if (xtr.GetAttribute("end_offset") != null)
@@ -141,9 +143,7 @@ namespace SabreTools.Helper
} }
else else
{ {
long temp = 0; rule.EndOffset = Convert.ToInt64(offset, 16);
Int64.TryParse(offset, out temp);
rule.EndOffset = temp;
} }
} }
if (xtr.GetAttribute("operation") != null) if (xtr.GetAttribute("operation") != null)
@@ -217,9 +217,7 @@ namespace SabreTools.Helper
} }
else else
{ {
long temp = 0; test.Offset = Convert.ToInt64(offset, 16);
Int64.TryParse(offset, out temp);
test.Offset = temp;
} }
} }
if (subreader.GetAttribute("value") != null) if (subreader.GetAttribute("value") != null)
@@ -269,9 +267,7 @@ namespace SabreTools.Helper
} }
else else
{ {
long temp = 0; test.Size = Convert.ToInt64(size, 16);
Int64.TryParse(size, out temp);
test.Size = temp;
} }
} }
if (subreader.GetAttribute("operator") != null) if (subreader.GetAttribute("operator") != null)
@@ -330,161 +326,150 @@ namespace SabreTools.Helper
} }
// Loop through and find a Skipper that has the right name // Loop through and find a Skipper that has the right name
Skipper matchedSkipper = new Skipper(); logger.User("Beginning search for matching header skip rules");
foreach (Skipper skipper in List) foreach (Skipper skipper in List)
{ {
if (!String.IsNullOrEmpty(skipper.Name) && skippername.ToLowerInvariant() == skipper.Name.ToLowerInvariant()) if (String.IsNullOrEmpty(skippername) || (!String.IsNullOrEmpty(skipper.Name) && skippername.ToLowerInvariant() == skipper.Name.ToLowerInvariant()))
{ {
matchedSkipper = skipper; // Loop through the rules until one is found that works
break; using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
}
}
// If we have a blank skipper, return a blank skipper rule
if (String.IsNullOrEmpty(matchedSkipper.Name) || matchedSkipper.Rules == null)
{
logger.Warning("No header skipper by the name of '" + skippername + "' can be found");
return skipperRule;
}
// Otherwise, loop through the rules until one is found that works
using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
{
logger.User("Beginning search for matching header skip rules");
foreach (SkipperRule rule in matchedSkipper.Rules)
{
// Always reset the stream back to the original place
br.BaseStream.Seek(0, SeekOrigin.Begin);
// For each rule, make sure it passes each test
bool success = true;
foreach (SkipperTest test in rule.Tests)
{ {
bool result = true; foreach (SkipperRule rule in skipper.Rules)
switch (test.Type)
{ {
case HeaderSkipTest.Data: // Always reset the stream back to the original place
// First seek to the correct position br.BaseStream.Seek(0, SeekOrigin.Begin);
if (test.Offset == null)
{
br.BaseStream.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
// Then read and compare bytewise // For each rule, make sure it passes each test
result = true; bool success = true;
for (int i = 0; i < test.Value.Length; i++) foreach (SkipperTest test in rule.Tests)
{
bool result = true;
switch (test.Type)
{ {
try case HeaderSkipTest.Data:
{ // First seek to the correct position
if (br.ReadByte() != test.Value[i]) if (test.Offset == null)
{ {
result = false; br.BaseStream.Seek(0, SeekOrigin.End);
break;
} }
} else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
catch {
{ br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
result = false; }
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
// Then read and compare bytewise
result = true;
for (int i = 0; i < test.Value.Length; i++)
{
try
{
if (br.ReadByte() != test.Value[i])
{
result = false;
break;
}
}
catch
{
result = false;
break;
}
}
// Return if the expected and actual results match
success &= (result == test.Result);
break; break;
} case HeaderSkipTest.Or:
} case HeaderSkipTest.Xor:
case HeaderSkipTest.And:
// First seek to the correct position
if (test.Offset == null)
{
br.BaseStream.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
// Return if the expected and actual results match result = true;
success &= (result == test.Result); try
break; {
case HeaderSkipTest.Or: // Then apply the mask if it exists
case HeaderSkipTest.Xor: byte[] read = br.ReadBytes(test.Mask.Length);
case HeaderSkipTest.And: byte[] masked = new byte[test.Mask.Length];
// First seek to the correct position for (int i = 0; i < read.Length; i++)
if (test.Offset == null) {
{ masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] :
br.BaseStream.Seek(0, SeekOrigin.End); (test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i])
} );
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length) }
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
result = true; // Finally, compare it against the value
try for (int i = 0; i < test.Value.Length; i++)
{ {
// Then apply the mask if it exists if (masked[i] != test.Value[i])
byte[] read = br.ReadBytes(test.Mask.Length); {
byte[] masked = new byte[test.Mask.Length]; result = false;
for (int i = 0; i < read.Length; i++) break;
{ }
masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] : }
(test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i]) }
); catch
}
// Finally, compare it against the value
for (int i = 0; i < test.Value.Length; i++)
{
if (masked[i] != test.Value[i])
{ {
result = false; result = false;
break;
} }
}
}
catch
{
result = false;
}
// Return if the expected and actual results match // Return if the expected and actual results match
success &= (result == test.Result); success &= (result == test.Result);
break; break;
case HeaderSkipTest.File: case HeaderSkipTest.File:
// First get the file size from stream // First get the file size from stream
long size = br.BaseStream.Length; long size = br.BaseStream.Length;
// If we have a null size, check that the size is a power of 2 // If we have a null size, check that the size is a power of 2
result = true; result = true;
if (test.Size == null) if (test.Size == null)
{ {
// http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2 // http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2
result = (((ulong)size & ((ulong)size - 1)) == 0); result = (((ulong)size & ((ulong)size - 1)) == 0);
} }
else if (test.Operator == HeaderSkipTestFileOperator.Less) else if (test.Operator == HeaderSkipTestFileOperator.Less)
{ {
result = (size < test.Size); result = (size < test.Size);
} }
else if (test.Operator == HeaderSkipTestFileOperator.Greater) else if (test.Operator == HeaderSkipTestFileOperator.Greater)
{ {
result = (size > test.Size); result = (size > test.Size);
} }
else if (test.Operator == HeaderSkipTestFileOperator.Equal) else if (test.Operator == HeaderSkipTestFileOperator.Equal)
{ {
result = (size == test.Size); result = (size == test.Size);
} }
// Return if the expected and actual results match // Return if the expected and actual results match
success &= (result == test.Result); success &= (result == test.Result);
break;
}
}
// If we still have a success, then return this rule
if (success)
{
logger.User("Matching rule found!");
skipperRule = rule;
break; break;
}
} }
} }
// If we still have a success, then return this rule
if (success)
{
logger.User("Matching rule found!");
skipperRule = rule;
break;
}
} }
} }
@@ -507,6 +492,8 @@ namespace SabreTools.Helper
/// <returns></returns> /// <returns></returns>
public static bool TransformFile(string input, string output, SkipperRule rule, Logger logger) public static bool TransformFile(string input, string output, SkipperRule rule, Logger logger)
{ {
bool success = true;
// If the input file doesn't exist, fail // If the input file doesn't exist, fail
if (!File.Exists(input)) if (!File.Exists(input))
{ {
@@ -515,9 +502,9 @@ namespace SabreTools.Helper
} }
// Create the output directory if it doesn't already // Create the output directory if it doesn't already
if (!Directory.Exists(Path.GetDirectoryName(output))) if (!Directory.Exists(System.IO.Path.GetDirectoryName(output)))
{ {
Directory.CreateDirectory(Path.GetDirectoryName(output)); Directory.CreateDirectory(System.IO.Path.GetDirectoryName(output));
} }
// If the sizes are wrong for the values, fail // If the sizes are wrong for the values, fail
@@ -533,17 +520,18 @@ namespace SabreTools.Helper
// Now read the proper part of the file and apply the rule // Now read the proper part of the file and apply the rule
try try
{ {
logger.User("Applying found rule to file '" + input + "'");
using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output))) using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output)))
using (BinaryReader br = new BinaryReader(File.OpenRead(input))) using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
{ {
// Seek to the beginning offset // Seek to the beginning offset
if (rule.StartOffset == null) if (rule.StartOffset == null)
{ {
return false; success = false;
} }
else if (Math.Abs((long)rule.StartOffset) > br.BaseStream.Length) else if (Math.Abs((long)rule.StartOffset) > br.BaseStream.Length)
{ {
return false; success = false;
} }
else if (rule.StartOffset > 0) else if (rule.StartOffset > 0)
{ {
@@ -555,65 +543,69 @@ namespace SabreTools.Helper
} }
// Then read and apply the operation as you go // Then read and apply the operation as you go
byte[] buffer = new byte[4]; if (success)
int pos = 0;
while (br.BaseStream.Position <= (rule.EndOffset != null ? rule.EndOffset : br.BaseStream.Length)
&& br.BaseStream.Position <= br.BaseStream.Length)
{ {
uint b = br.ReadUInt32(); byte[] buffer = new byte[4];
switch (rule.Operation) int pos = 0;
while (br.BaseStream.Position < (rule.EndOffset != null ? rule.EndOffset : br.BaseStream.Length)
&& br.BaseStream.Position < br.BaseStream.Length)
{ {
case HeaderSkipOperation.Bitswap: byte b = br.ReadByte();
// http://stackoverflow.com/questions/3587826/is-there-a-built-in-function-to-reverse-bit-order switch (rule.Operation)
uint r = b; {
int s = 7; case HeaderSkipOperation.Bitswap:
for (b >>= 1; b != 0; b >>= 1) // http://stackoverflow.com/questions/3587826/is-there-a-built-in-function-to-reverse-bit-order
{ uint r = b;
r <<= 1; int s = 7;
r |= (byte)(b & 1); for (b >>= 1; b != 0; b >>= 1)
s--; {
} r <<= 1;
r <<= s; r |= (byte)(b & 1);
buffer[pos] = (byte)r; s--;
break; }
case HeaderSkipOperation.Byteswap: r <<= s;
if (pos % 2 == 1) buffer[pos] = (byte)r;
{ break;
buffer[pos - 1] = (byte)b; case HeaderSkipOperation.Byteswap:
} if (pos % 2 == 1)
if (pos % 2 == 0) {
{ buffer[pos - 1] = b;
buffer[pos + 1] = (byte)b; }
} if (pos % 2 == 0)
break; {
case HeaderSkipOperation.Wordswap: buffer[pos + 1] = b;
buffer[3 - pos] = (byte)b; }
break; break;
case HeaderSkipOperation.WordByteswap: case HeaderSkipOperation.Wordswap:
buffer[(pos + 2) % 4] = (byte)b; buffer[3 - pos] = b;
break; break;
case HeaderSkipOperation.None: case HeaderSkipOperation.WordByteswap:
default: buffer[(pos + 2) % 4] = b;
buffer[pos] = (byte)b; break;
break; case HeaderSkipOperation.None:
default:
buffer[pos] = b;
break;
}
// Set the buffer position to default write to
pos = (pos + 1) % 4;
// If we filled a buffer, flush to the stream
if (pos == 0)
{
bw.Write(buffer);
bw.Flush();
buffer = new byte[4];
}
} }
// If there's anything more in the buffer, write only the left bits
// Set the buffer position to default write to for (int i = 0; i < pos; i++)
pos = (pos + 1) % 4;
// If we filled a buffer, flush to the stream
if (pos == 0)
{ {
bw.Write(buffer); bw.Write(buffer[i]);
buffer = new byte[4];
} }
} }
// If there's anything more in the buffer, write only the left bits
for (int i = 0; i < pos; i++)
{
bw.Write(buffer[i]);
}
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -622,7 +614,20 @@ namespace SabreTools.Helper
return false; return false;
} }
return true; // If the output file has size 0, delete it
if (new FileInfo(output).Length == 0)
{
try
{
File.Delete(output);
}
catch
{
// Don't log this file deletion error
}
}
return success;
} }
#endregion #endregion
@@ -658,7 +663,7 @@ namespace SabreTools.Helper
XmlDocument doc = new XmlDocument(); XmlDocument doc = new XmlDocument();
try try
{ {
doc.LoadXml(File.ReadAllText(Path.Combine(_skippersPath, skipper + ".xml"))); doc.LoadXml(File.ReadAllText(System.IO.Path.Combine(Path, skipper + ".xml")));
} }
catch (XmlException ex) catch (XmlException ex)
{ {
@@ -744,7 +749,6 @@ namespace SabreTools.Helper
} }
catch catch
{ {
logger.Warning("The mapping for '" + test.ToString() + "' cannot be found!");
continue; continue;
} }

13
SimpleSort/SimpleSort.cs
View File

@@ -375,14 +375,15 @@ namespace SabreTools
ArchiveTools.WriteToArchive(input, _outdir, found); ArchiveTools.WriteToArchive(input, _outdir, found);
} }
// Now get the headerless file if it exists // Now get the transformed file if it exists
int hs = 0; SkipperRule rule = Skippers.MatchesSkipper(input, "", _logger);
Skippers.GetFileHeaderType(input, out hs, _logger);
if (hs > 0) // If we have have a non-empty rule, apply it
if (rule.Tests != null && rule.Tests.Count != 0)
{ {
// Otherwise, apply the rule ot the file
string newinput = input + ".new"; string newinput = input + ".new";
_logger.Log("Creating unheadered file: '" + newinput + "'"); Skippers.TransformFile(input, input + ".new", rule, _logger);
Output.RemoveBytesFromFile(input, newinput, hs, 0);
Rom drom = RomTools.GetSingleFileInfo(newinput); Rom drom = RomTools.GetSingleFileInfo(newinput);
// If we have a blank RomData, it's an error // If we have a blank RomData, it's an error