From c70f0d72d28b03205c62f98a5e1c70ac077d59ec Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 3 Sep 2025 08:35:19 -0400 Subject: [PATCH] Downstream potential fixes for testing --- BinaryObjectScanner/FileType/Executable.cs | 201 ++++++++++++++++++- BinaryObjectScanner/Protection/ActiveMARK.cs | 4 +- BinaryObjectScanner/Protection/AlphaROM.cs | 4 +- BinaryObjectScanner/Protection/CopyX.cs | 4 +- BinaryObjectScanner/Protection/SecuROM.cs | 4 +- 5 files changed, 203 insertions(+), 14 deletions(-) diff --git a/BinaryObjectScanner/FileType/Executable.cs b/BinaryObjectScanner/FileType/Executable.cs index 9d073096..7db6ec82 100644 --- a/BinaryObjectScanner/FileType/Executable.cs +++ b/BinaryObjectScanner/FileType/Executable.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using BinaryObjectScanner.Data; using BinaryObjectScanner.Interfaces; using SabreTools.IO.Extensions; @@ -118,14 +119,14 @@ namespace BinaryObjectScanner.FileType public bool Extract(Stream? stream, string file, string outDir, bool includeDebug) { // Create the wrapper - var exe = WrapperFactory.CreateExecutableWrapper(stream); - if (exe == null) + var wrapper = WrapperFactory.CreateExecutableWrapper(stream); + if (wrapper == null) return false; // Extract all files bool extractAny = false; Directory.CreateDirectory(outDir); - if (exe is PortableExecutable pex) + if (wrapper is PortableExecutable pex) { if (new Packer.CExe().CheckExecutable(file, pex, includeDebug) != null) extractAny |= pex.ExtractCExe(outDir, includeDebug); @@ -139,12 +140,10 @@ namespace BinaryObjectScanner.FileType if (new Packer.WiseInstaller().CheckExecutable(file, pex, includeDebug) != null) extractAny |= pex.ExtractWise(outDir, includeDebug); } - else if (exe is NewExecutable nex) + else if (wrapper is NewExecutable nex) { if (new Packer.EmbeddedFile().CheckExecutable(file, nex, includeDebug) != null) - { extractAny |= nex.ExtractFromOverlay(outDir, includeDebug); - } if (new Packer.WiseInstaller().CheckExecutable(file, nex, includeDebug) != null) extractAny |= nex.ExtractWise(outDir, includeDebug); @@ -236,6 +235,196 @@ namespace BinaryObjectScanner.FileType return protections; } + #endregion + + #region Helpers -- Remove when dependent libraries updated + + /// + /// Map to contain overlay strings to avoid rereading + /// + private static readonly Dictionary> _overlayStrings = []; + + /// + /// Lock object for + /// + private static readonly object _overlayStringsLock = new(); + + /// + /// Overlay strings, if they exist + /// + public static List? GetOverlayStrings(NewExecutable nex) + { + lock (_overlayStringsLock) + { + // Use the cached data if possible + if (_overlayStrings.TryGetValue(nex, out var strings)) + return strings; + + // Get the available source length, if possible + long dataLength = nex.Length; + if (dataLength == -1) + return null; + + // If a required property is missing + if (nex.Header == null || nex.SegmentTable == null || nex.ResourceTable?.ResourceTypes == null) + return null; + + // Get the overlay data, if possible + byte[]? overlayData = nex.OverlayData; + if (overlayData == null || overlayData.Length == 0) + { + _overlayStrings[nex] = []; + return []; + } + + // Otherwise, cache and return the strings + _overlayStrings[nex] = ReadStringsFrom(overlayData, charLimit: 3) ?? []; + return _overlayStrings[nex]; + } + } + + /// + /// Overlay strings, if they exist + /// + public static List? GetOverlayStrings(PortableExecutable pex) + { + lock (_overlayStringsLock) + { + // Use the cached data if possible + if (_overlayStrings.TryGetValue(pex, out var strings)) + return strings; + + // Get the available source length, if possible + long dataLength = pex.Length; + if (dataLength == -1) + return null; + + // If the section table is missing + if (pex.SectionTable == null) + return null; + + // Get the overlay data, if possible + byte[]? overlayData = pex.OverlayData; + if (overlayData == null || overlayData.Length == 0) + { + _overlayStrings[pex] = []; + return []; + } + + // Otherwise, cache and return the strings + _overlayStrings[pex] = ReadStringsFrom(overlayData, charLimit: 3) ?? []; + return _overlayStrings[pex]; + } + } + + /// + /// Read string data from the source + /// + /// Number of characters needed to be a valid string, default 5 + /// String list containing the requested data, null on error + private static List? ReadStringsFrom(byte[]? input, int charLimit = 5) + { + // Validate the data + if (input == null || input.Length == 0) + return null; + + // Check for ASCII strings + var asciiStrings = ReadStringsWithEncoding(input, charLimit, Encoding.ASCII); + + // Check for UTF-8 strings + // We are limiting the check for Unicode characters with a second byte of 0x00 for now + var utf8Strings = ReadStringsWithEncoding(input, charLimit, Encoding.UTF8); + + // Check for Unicode strings + // We are limiting the check for Unicode characters with a second byte of 0x00 for now + var unicodeStrings = ReadStringsWithEncoding(input, charLimit, Encoding.Unicode); + + // Ignore duplicate strings across encodings + List sourceStrings = [.. asciiStrings, .. utf8Strings, .. unicodeStrings]; + + // Sort the strings and return + sourceStrings.Sort(); + return sourceStrings; + } + + /// + /// Read string data from the source with an encoding + /// + /// Byte array representing the source data + /// Number of characters needed to be a valid string + /// Character encoding to use for checking + /// String list containing the requested data, empty on error + /// + /// This method has a couple of notable implementation details: + /// - Strings can only have a maximum of 64 characters + /// - Characters that fall outside of the extended ASCII set will be unused + /// +#if NET20 + private static List ReadStringsWithEncoding(byte[]? bytes, int charLimit, Encoding encoding) +#else + private static HashSet ReadStringsWithEncoding(byte[]? bytes, int charLimit, Encoding encoding) +#endif + { + if (bytes == null || bytes.Length == 0) + return []; + if (charLimit <= 0 || charLimit > bytes.Length) + return []; + + // Create the string set to return +#if NET20 + var strings = new List(); +#else + var strings = new HashSet(); +#endif + + // Open the text reader with the correct encoding + using var ms = new MemoryStream(bytes); + using var reader = new StreamReader(ms, encoding); + + // Create a string builder for the loop + var sb = new StringBuilder(); + + // Check for strings + long lastOffset = 0; + while (!reader.EndOfStream) + { + // Read the next character from the stream + char c = (char)reader.Read(); + + // If the character is invalid + if (char.IsControl(c) || (c & 0xFF00) != 0) + { + // Seek to the end of the last found string + string str = sb.ToString(); + lastOffset += encoding.GetByteCount(str) + 1; + ms.Seek(lastOffset, SeekOrigin.Begin); + reader.DiscardBufferedData(); + + // Add the string if long enough + if (str.Length >= charLimit) + strings.Add(str); + + // Clear the builder and continue +#if NET20 || NET35 + sb = new(); +#else + sb.Clear(); +#endif + continue; + } + + // Otherwise, add the character to the builder and continue + sb.Append(c); + } + + // Handle any remaining data + if (sb.Length >= charLimit) + strings.Add(sb.ToString()); + + return strings; + } + + #endregion } } diff --git a/BinaryObjectScanner/Protection/ActiveMARK.cs b/BinaryObjectScanner/Protection/ActiveMARK.cs index 9fde52d0..acbf599e 100644 --- a/BinaryObjectScanner/Protection/ActiveMARK.cs +++ b/BinaryObjectScanner/Protection/ActiveMARK.cs @@ -62,9 +62,9 @@ namespace BinaryObjectScanner.Protection } // Get the overlay data, if it exists - if (pex.OverlayStrings != null) + if (FileType.Executable.GetOverlayStrings(pex) != null) { - if (pex.OverlayStrings.Exists(s => s.Contains("TMSAMVOH"))) + if (FileType.Executable.GetOverlayStrings(pex)!.Exists(s => s.Contains("TMSAMVOH"))) return "ActiveMARK"; } diff --git a/BinaryObjectScanner/Protection/AlphaROM.cs b/BinaryObjectScanner/Protection/AlphaROM.cs index 6786b729..a13a559f 100644 --- a/BinaryObjectScanner/Protection/AlphaROM.cs +++ b/BinaryObjectScanner/Protection/AlphaROM.cs @@ -75,10 +75,10 @@ namespace BinaryObjectScanner.Protection } // Get the overlay data, if it exists - if (pex.OverlayStrings != null) + if (FileType.Executable.GetOverlayStrings(pex) != null) { // Found in Redump entry 84122. - if (pex.OverlayStrings.Exists(s => s.Contains("SETTEC0000"))) + if (FileType.Executable.GetOverlayStrings(pex)!.Exists(s => s.Contains("SETTEC0000"))) return "Alpha-ROM"; } diff --git a/BinaryObjectScanner/Protection/CopyX.cs b/BinaryObjectScanner/Protection/CopyX.cs index f14de804..44e85bfa 100644 --- a/BinaryObjectScanner/Protection/CopyX.cs +++ b/BinaryObjectScanner/Protection/CopyX.cs @@ -66,7 +66,7 @@ namespace BinaryObjectScanner.Protection if (sections == null) return null; - if (pex.OverlayStrings != null) + if (FileType.Executable.GetOverlayStrings(pex) != null) { // Checks if main executable contains reference to optgraph.dll. // This might be better removed later, as Redump ID 82475 is a false positive, and also doesn't actually @@ -76,7 +76,7 @@ namespace BinaryObjectScanner.Protection // TODO: This might need to check every single section. Unsure until more samples are acquired. // TODO: TKKG also has an NE 3.1x executable with a reference. This can be added later. // Samples: Redump ID 108150 - if (pex.OverlayStrings.Exists(s => s.Contains("optgraph.dll"))) + if (FileType.Executable.GetOverlayStrings(pex)!.Exists(s => s.Contains("optgraph.dll"))) return "copy-X [Check disc for physical ring]"; } diff --git a/BinaryObjectScanner/Protection/SecuROM.cs b/BinaryObjectScanner/Protection/SecuROM.cs index 0f59c174..d7209fe0 100644 --- a/BinaryObjectScanner/Protection/SecuROM.cs +++ b/BinaryObjectScanner/Protection/SecuROM.cs @@ -64,9 +64,9 @@ namespace BinaryObjectScanner.Protection return $"SecuROM SLL Protected (for SecuROM v8.x)"; // Search after the last section - if (pex.OverlayStrings != null) + if (FileType.Executable.GetOverlayStrings(pex) != null) { - if (pex.OverlayStrings.Exists(s => s == "AddD")) + if (FileType.Executable.GetOverlayStrings(pex)!.Exists(s => s == "AddD")) return $"SecuROM {GetV4Version(pex)}"; }