[Skippers] Add stream-based versions of check and transform; convert current versions to use this internally

This commit is contained in:
Matt Nadareski
2016-09-17 17:30:25 -07:00
parent e395ad2301
commit 0b52d2502e

View File

@@ -311,162 +311,181 @@ namespace SabreTools.Helper
/// Get the SkipperRule associated with a given file /// Get the SkipperRule associated with a given file
/// </summary> /// </summary>
/// <param name="input">Name of the file to be checked</param> /// <param name="input">Name of the file to be checked</param>
/// <param name="skippername">Name of the skipper to be used</param> /// <param name="skipperName">Name of the skipper to be used</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
/// <returns>The SkipperRule that matched the file</returns> /// <returns>The SkipperRule that matched the file</returns>
public static SkipperRule MatchesSkipper(string input, string skippername, Logger logger) public static SkipperRule MatchesSkipper(string input, string skipperName, Logger logger)
{ {
SkipperRule skipperRule = new SkipperRule();
// If the file doesn't exist, return a blank skipper rule // If the file doesn't exist, return a blank skipper rule
if (!File.Exists(input)) if (!File.Exists(input))
{ {
logger.Error("The file '" + input + "' does not exist so it cannot be tested"); logger.Error("The file '" + input + "' does not exist so it cannot be tested");
return skipperRule; return new SkipperRule();
} }
return MatchesSkipper(File.OpenRead(input), skipperName, logger);
}
/// <summary>
/// Get the SkipperRule associated with a given stream
/// </summary>
/// <param name="input">Name of the file to be checked</param>
/// <param name="skipperName">Name of the skipper to be used</param>
/// <param name="logger">Logger object for file and console output</param>
/// <param name="keepOpen">True if the underlying stream should be kept open, false otherwise</param>
/// <returns>The SkipperRule that matched the file</returns>
public static SkipperRule MatchesSkipper(Stream input, string skipperName, Logger logger, bool keepOpen = false)
{
SkipperRule skipperRule = new SkipperRule();
// Loop through and find a Skipper that has the right name // Loop through and find a Skipper that has the right name
logger.Log("Beginning search for matching header skip rules"); logger.Log("Beginning search for matching header skip rules");
foreach (Skipper skipper in List) foreach (Skipper skipper in List)
{ {
if (String.IsNullOrEmpty(skippername) || (!String.IsNullOrEmpty(skipper.Name) && skippername.ToLowerInvariant() == skipper.Name.ToLowerInvariant())) if (String.IsNullOrEmpty(skipperName) || (!String.IsNullOrEmpty(skipper.Name) && skipperName.ToLowerInvariant() == skipper.Name.ToLowerInvariant()))
{ {
// Loop through the rules until one is found that works // Loop through the rules until one is found that works
using (BinaryReader br = new BinaryReader(File.OpenRead(input))) BinaryReader br = new BinaryReader(input);
foreach (SkipperRule rule in skipper.Rules)
{ {
foreach (SkipperRule rule in skipper.Rules) // Always reset the stream back to the original place
br.BaseStream.Seek(0, SeekOrigin.Begin);
// For each rule, make sure it passes each test
bool success = true;
foreach (SkipperTest test in rule.Tests)
{ {
// Always reset the stream back to the original place bool result = true;
br.BaseStream.Seek(0, SeekOrigin.Begin); switch (test.Type)
// For each rule, make sure it passes each test
bool success = true;
foreach (SkipperTest test in rule.Tests)
{ {
bool result = true; case HeaderSkipTest.Data:
switch (test.Type) // First seek to the correct position
{ if (test.Offset == null)
case HeaderSkipTest.Data: {
// First seek to the correct position br.BaseStream.Seek(0, SeekOrigin.End);
if (test.Offset == null) }
{ else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
br.BaseStream.Seek(0, SeekOrigin.End); {
} br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length) }
{ else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin); {
} br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length) }
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
// Then read and compare bytewise // Then read and compare bytewise
result = true; result = true;
for (int i = 0; i < test.Value.Length; i++) for (int i = 0; i < test.Value.Length; i++)
{
try
{ {
try if (br.ReadByte() != test.Value[i])
{
if (br.ReadByte() != test.Value[i])
{
result = false;
break;
}
}
catch
{ {
result = false; result = false;
break; break;
} }
} }
// Return if the expected and actual results match
success &= (result == test.Result);
break;
case HeaderSkipTest.Or:
case HeaderSkipTest.Xor:
case HeaderSkipTest.And:
// First seek to the correct position
if (test.Offset == null)
{
br.BaseStream.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
result = true;
try
{
// Then apply the mask if it exists
byte[] read = br.ReadBytes(test.Mask.Length);
byte[] masked = new byte[test.Mask.Length];
for (int i = 0; i < read.Length; i++)
{
masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] :
(test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i])
);
}
// Finally, compare it against the value
for (int i = 0; i < test.Value.Length; i++)
{
if (masked[i] != test.Value[i])
{
result = false;
break;
}
}
}
catch catch
{ {
result = false; result = false;
break;
}
}
// Return if the expected and actual results match
success &= (result == test.Result);
break;
case HeaderSkipTest.Or:
case HeaderSkipTest.Xor:
case HeaderSkipTest.And:
// First seek to the correct position
if (test.Offset == null)
{
br.BaseStream.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= br.BaseStream.Length)
{
br.BaseStream.Seek((long)test.Offset, SeekOrigin.End);
}
result = true;
try
{
// Then apply the mask if it exists
byte[] read = br.ReadBytes(test.Mask.Length);
byte[] masked = new byte[test.Mask.Length];
for (int i = 0; i < read.Length; i++)
{
masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] :
(test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i])
);
} }
// Return if the expected and actual results match // Finally, compare it against the value
success &= (result == test.Result); for (int i = 0; i < test.Value.Length; i++)
break; {
case HeaderSkipTest.File: if (masked[i] != test.Value[i])
// First get the file size from stream {
long size = br.BaseStream.Length; result = false;
break;
}
}
}
catch
{
result = false;
}
// If we have a null size, check that the size is a power of 2 // Return if the expected and actual results match
result = true; success &= (result == test.Result);
if (test.Size == null) break;
{ case HeaderSkipTest.File:
// http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2 // First get the file size from stream
result = (((ulong)size & ((ulong)size - 1)) == 0); long size = br.BaseStream.Length;
}
else if (test.Operator == HeaderSkipTestFileOperator.Less)
{
result = (size < test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Greater)
{
result = (size > test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Equal)
{
result = (size == test.Size);
}
// Return if the expected and actual results match // If we have a null size, check that the size is a power of 2
success &= (result == test.Result); result = true;
break; if (test.Size == null)
} {
// http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2
result = (((ulong)size & ((ulong)size - 1)) == 0);
}
else if (test.Operator == HeaderSkipTestFileOperator.Less)
{
result = (size < test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Greater)
{
result = (size > test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Equal)
{
result = (size == test.Size);
}
// Return if the expected and actual results match
success &= (result == test.Result);
break;
} }
}
// If we still have a success, then return this rule // If we're not keeping the stream open, dispose of the binary reader
if (success) if (!keepOpen)
{ {
logger.User(" Matching rule found!"); br.Close();
return rule; br.Dispose();
} }
// If we still have a success, then return this rule
if (success)
{
logger.User(" Matching rule found!");
return rule;
} }
} }
} }
@@ -488,7 +507,7 @@ namespace SabreTools.Helper
/// <param name="output">Output file name</param> /// <param name="output">Output file name</param>
/// <param name="rule">SkipperRule to apply to the file</param> /// <param name="rule">SkipperRule to apply to the file</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
/// <returns></returns> /// <returns>True if the file was transformed properly, false otherwise</returns>
public static bool TransformFile(string input, string output, SkipperRule rule, Logger logger) public static bool TransformFile(string input, string output, SkipperRule rule, Logger logger)
{ {
bool success = true; bool success = true;
@@ -506,112 +525,8 @@ namespace SabreTools.Helper
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(output)); Directory.CreateDirectory(System.IO.Path.GetDirectoryName(output));
} }
// If the sizes are wrong for the values, fail logger.User("Attempting to apply rule to '" + input + "'");
long extsize = new FileInfo(input).Length; success = TransformStream(File.OpenRead(input), File.OpenWrite(output), rule, logger);
if ((rule.Operation > HeaderSkipOperation.Bitswap && (extsize % 2) != 0)
|| (rule.Operation > HeaderSkipOperation.Byteswap && (extsize % 4) != 0)
|| (rule.Operation > HeaderSkipOperation.Bitswap && (rule.StartOffset == null || rule.StartOffset % 2 == 0)))
{
logger.Error("The file did not have the correct size to be transformed!");
return false;
}
// Now read the proper part of the file and apply the rule
try
{
logger.User("Applying found rule to file '" + input + "'");
using (BinaryWriter bw = new BinaryWriter(File.OpenWrite(output)))
using (BinaryReader br = new BinaryReader(File.OpenRead(input)))
{
// Seek to the beginning offset
if (rule.StartOffset == null)
{
success = false;
}
else if (Math.Abs((long)rule.StartOffset) > br.BaseStream.Length)
{
success = false;
}
else if (rule.StartOffset > 0)
{
br.BaseStream.Seek((long)rule.StartOffset, SeekOrigin.Begin);
}
else if (rule.StartOffset < 0)
{
br.BaseStream.Seek((long)rule.StartOffset, SeekOrigin.End);
}
// Then read and apply the operation as you go
if (success)
{
byte[] buffer = new byte[4];
int pos = 0;
while (br.BaseStream.Position < (rule.EndOffset != null ? rule.EndOffset : br.BaseStream.Length)
&& br.BaseStream.Position < br.BaseStream.Length)
{
byte b = br.ReadByte();
switch (rule.Operation)
{
case HeaderSkipOperation.Bitswap:
// http://stackoverflow.com/questions/3587826/is-there-a-built-in-function-to-reverse-bit-order
uint r = b;
int s = 7;
for (b >>= 1; b != 0; b >>= 1)
{
r <<= 1;
r |= (byte)(b & 1);
s--;
}
r <<= s;
buffer[pos] = (byte)r;
break;
case HeaderSkipOperation.Byteswap:
if (pos % 2 == 1)
{
buffer[pos - 1] = b;
}
if (pos % 2 == 0)
{
buffer[pos + 1] = b;
}
break;
case HeaderSkipOperation.Wordswap:
buffer[3 - pos] = b;
break;
case HeaderSkipOperation.WordByteswap:
buffer[(pos + 2) % 4] = b;
break;
case HeaderSkipOperation.None:
default:
buffer[pos] = b;
break;
}
// Set the buffer position to default write to
pos = (pos + 1) % 4;
// If we filled a buffer, flush to the stream
if (pos == 0)
{
bw.Write(buffer);
bw.Flush();
buffer = new byte[4];
}
}
// If there's anything more in the buffer, write only the left bits
for (int i = 0; i < pos; i++)
{
bw.Write(buffer[i]);
}
}
}
}
catch (Exception ex)
{
logger.Error(ex.ToString());
return false;
}
// If the output file has size 0, delete it // If the output file has size 0, delete it
if (new FileInfo(output).Length == 0) if (new FileInfo(output).Length == 0)
@@ -629,6 +544,146 @@ namespace SabreTools.Helper
return success; return success;
} }
/// <summary>
/// Transform an input stream using the given rule
/// </summary>
/// <param name="input">Input stream</param>
/// <param name="output">Output stream</param>
/// <param name="rule">SkipperRule to apply to the stream</param>
/// <param name="logger">Logger object for file and console output</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <param name="keepWriteOpen">True if the underlying write stream should be kept open, false otherwise</param>
/// <returns>True if the file was transformed properly, false otherwise</returns>
public static bool TransformStream(Stream input, Stream output, SkipperRule rule, Logger logger, bool keepReadOpen = false, bool keepWriteOpen = false)
{
bool success = true;
// If the sizes are wrong for the values, fail
long extsize = input.Length;
if ((rule.Operation > HeaderSkipOperation.Bitswap && (extsize % 2) != 0)
|| (rule.Operation > HeaderSkipOperation.Byteswap && (extsize % 4) != 0)
|| (rule.Operation > HeaderSkipOperation.Bitswap && (rule.StartOffset == null || rule.StartOffset % 2 == 0)))
{
logger.Error("The stream did not have the correct size to be transformed!");
return false;
}
// Now read the proper part of the file and apply the rule
BinaryWriter bw = null;
BinaryReader br = null;
try
{
logger.User("Applying found rule to input stream");
bw = new BinaryWriter(output);
br = new BinaryReader(input);
// Seek to the beginning offset
if (rule.StartOffset == null)
{
success = false;
}
else if (Math.Abs((long)rule.StartOffset) > br.BaseStream.Length)
{
success = false;
}
else if (rule.StartOffset > 0)
{
br.BaseStream.Seek((long)rule.StartOffset, SeekOrigin.Begin);
}
else if (rule.StartOffset < 0)
{
br.BaseStream.Seek((long)rule.StartOffset, SeekOrigin.End);
}
// Then read and apply the operation as you go
if (success)
{
byte[] buffer = new byte[4];
int pos = 0;
while (br.BaseStream.Position < (rule.EndOffset != null ? rule.EndOffset : br.BaseStream.Length)
&& br.BaseStream.Position < br.BaseStream.Length)
{
byte b = br.ReadByte();
switch (rule.Operation)
{
case HeaderSkipOperation.Bitswap:
// http://stackoverflow.com/questions/3587826/is-there-a-built-in-function-to-reverse-bit-order
uint r = b;
int s = 7;
for (b >>= 1; b != 0; b >>= 1)
{
r <<= 1;
r |= (byte)(b & 1);
s--;
}
r <<= s;
buffer[pos] = (byte)r;
break;
case HeaderSkipOperation.Byteswap:
if (pos % 2 == 1)
{
buffer[pos - 1] = b;
}
if (pos % 2 == 0)
{
buffer[pos + 1] = b;
}
break;
case HeaderSkipOperation.Wordswap:
buffer[3 - pos] = b;
break;
case HeaderSkipOperation.WordByteswap:
buffer[(pos + 2) % 4] = b;
break;
case HeaderSkipOperation.None:
default:
buffer[pos] = b;
break;
}
// Set the buffer position to default write to
pos = (pos + 1) % 4;
// If we filled a buffer, flush to the stream
if (pos == 0)
{
bw.Write(buffer);
bw.Flush();
buffer = new byte[4];
}
}
// If there's anything more in the buffer, write only the left bits
for (int i = 0; i < pos; i++)
{
bw.Write(buffer[i]);
}
}
}
catch (Exception ex)
{
logger.Error(ex.ToString());
return false;
}
finally
{
// If we're not keeping the read stream open, dispose of the binary reader
if (!keepReadOpen)
{
br?.Close();
br?.Dispose();
}
// If we're not keeping the write stream open, dispose of the binary reader
if (!keepWriteOpen)
{
bw?.Close();
bw?.Dispose();
}
}
return success;
}
#endregion #endregion
#region Header Skips (old) #region Header Skips (old)