using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using BinaryObjectScanner.FileType; using BinaryObjectScanner.Interfaces; using BinaryObjectScanner.Utilities; using BinaryObjectScanner.Wrappers; using static BinaryObjectScanner.Utilities.Dictionary; namespace BurnOutSharp { public class Scanner { #region Options /// public bool ScanArchives => options?.ScanArchives ?? false; /// public bool ScanContents => options?.ScanContents ?? false; /// public bool ScanGameEngines => options?.ScanGameEngines ?? false; /// public bool ScanPackers => options?.ScanPackers ?? false; /// public bool ScanPaths => options?.ScanPaths ?? false; /// public bool IncludeDebug => options?.IncludeDebug ?? false; /// /// Options object for configuration /// private readonly Options options; #endregion /// /// Optional progress callback during scanning /// private readonly IProgress fileProgress; /// /// Constructor /// /// Enable scanning archive contents /// Enable including content detections in output /// Enable including game engines in output /// Enable including packers in output /// Enable including path detections in output /// Enable including debug information /// Optional progress callback public Scanner(bool scanArchives, bool scanContents, bool scanGameEngines, bool scanPackers, bool scanPaths, bool includeDebug, IProgress fileProgress = null) { this.options = new Options { ScanArchives = scanArchives, ScanContents = scanContents, ScanGameEngines = scanGameEngines, ScanPackers = scanPackers, ScanPaths = scanPaths, IncludeDebug = includeDebug, }; this.fileProgress = fileProgress; // Register the codepages Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); } #region Scanning /// /// Scan a single path and get all found protections /// /// Path to scan /// Dictionary of list of strings representing the found protections public ConcurrentDictionary> GetProtections(string path) { return GetProtections(new List { path }); } /// /// Scan the list of paths and get all found protections /// /// Dictionary of list of strings representing the found protections public ConcurrentDictionary> GetProtections(List paths) { // If we have no paths, we can't scan if (paths == null || !paths.Any()) return null; // Set a starting starting time for debug output DateTime startTime = DateTime.UtcNow; // Checkpoint this.fileProgress?.Report(new ProtectionProgress(null, 0, null)); // Temp variables for reporting string tempFilePath = Path.GetTempPath(); string tempFilePathWithGuid = Path.Combine(tempFilePath, Guid.NewGuid().ToString()); // Loop through each path and get the returned values var protections = new ConcurrentDictionary>(); foreach (string path in paths) { // Directories scan each internal file individually if (Directory.Exists(path)) { // Enumerate all files at first for easier access var files = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).ToList(); // Scan for path-detectable protections if (ScanPaths) { var directoryPathProtections = Handler.HandlePathChecks(path, files); AppendToDictionary(protections, directoryPathProtections); } // Scan each file in directory separately for (int i = 0; i < files.Count; i++) { // Get the current file string file = files.ElementAt(i); // Get the reportable file name string reportableFileName = file; if (reportableFileName.StartsWith(tempFilePath)) reportableFileName = reportableFileName.Substring(tempFilePathWithGuid.Length); // Checkpoint this.fileProgress?.Report(new ProtectionProgress(reportableFileName, i / (float)files.Count, "Checking file" + (file != reportableFileName ? " from archive" : string.Empty))); // Scan for path-detectable protections if (ScanPaths) { var filePathProtections = Handler.HandlePathChecks(file, files: null); AppendToDictionary(protections, filePathProtections); } // Scan for content-detectable protections var fileProtections = GetInternalProtections(file); if (fileProtections != null && fileProtections.Any()) { foreach (string key in fileProtections.Keys) { if (!protections.ContainsKey(key)) protections[key] = new ConcurrentQueue(); protections[key].AddRange(fileProtections[key]); } } // Checkpoint protections.TryGetValue(file, out ConcurrentQueue fullProtectionList); string fullProtection = (fullProtectionList != null && fullProtectionList.Any() ? string.Join(", ", fullProtectionList) : null); this.fileProgress?.Report(new ProtectionProgress(reportableFileName, (i + 1) / (float)files.Count, fullProtection ?? string.Empty)); } } // Scan a single file by itself else if (File.Exists(path)) { // Get the reportable file name string reportableFileName = path; if (reportableFileName.StartsWith(tempFilePath)) reportableFileName = reportableFileName.Substring(tempFilePathWithGuid.Length); // Checkpoint this.fileProgress?.Report(new ProtectionProgress(reportableFileName, 0, "Checking file" + (path != reportableFileName ? " from archive" : string.Empty))); // Scan for path-detectable protections if (ScanPaths) { var filePathProtections = Handler.HandlePathChecks(path, files: null); AppendToDictionary(protections, filePathProtections); } // Scan for content-detectable protections var fileProtections = GetInternalProtections(path); if (fileProtections != null && fileProtections.Any()) { foreach (string key in fileProtections.Keys) { if (!protections.ContainsKey(key)) protections[key] = new ConcurrentQueue(); protections[key].AddRange(fileProtections[key]); } } // Checkpoint protections.TryGetValue(path, out ConcurrentQueue fullProtectionList); string fullProtection = (fullProtectionList != null && fullProtectionList.Any() ? string.Join(", ", fullProtectionList) : null); this.fileProgress?.Report(new ProtectionProgress(reportableFileName, 1, fullProtection ?? string.Empty)); } // Throw on an invalid path else { Console.WriteLine($"{path} is not a directory or file, skipping..."); //throw new FileNotFoundException($"{path} is not a directory or file, skipping..."); } } // Clear out any empty keys ClearEmptyKeys(protections); // If we're in debug, output the elasped time to console if (IncludeDebug) Console.WriteLine($"Time elapsed: {DateTime.UtcNow.Subtract(startTime)}"); return protections; } /// /// Get the content-detectable protections associated with a single path /// /// Path to the file to scan /// Dictionary of list of strings representing the found protections private ConcurrentDictionary> GetInternalProtections(string file) { // Quick sanity check before continuing if (!File.Exists(file)) return null; // Open the file and begin scanning try { using (FileStream fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) { return GetInternalProtections(file, fs); } } catch (Exception ex) { if (IncludeDebug) Console.WriteLine(ex); var protections = new ConcurrentDictionary>(); AppendToDictionary(protections, file, IncludeDebug ? ex.ToString() : "[Exception opening file, please try again]"); ClearEmptyKeys(protections); return protections; } } /// /// Get the content-detectable protections associated with a single path /// /// Name of the source file of the stream, for tracking /// Stream to scan the contents of /// Dictionary of list of strings representing the found protections private ConcurrentDictionary> GetInternalProtections(string fileName, Stream stream) { // Quick sanity check before continuing if (stream == null || !stream.CanRead || !stream.CanSeek) return null; // Initialize the protections found var protections = new ConcurrentDictionary>(); // Get the extension for certain checks string extension = Path.GetExtension(fileName).ToLower().TrimStart('.'); // Open the file and begin scanning try { // Get the first 16 bytes for matching byte[] magic = new byte[16]; try { stream.Read(magic, 0, 16); stream.Seek(0, SeekOrigin.Begin); } catch (Exception ex) { if (IncludeDebug) Console.WriteLine(ex); return null; } // Get the file type either from magic number or extension SupportedFileType fileType = FileTypes.GetFileType(magic); if (fileType == SupportedFileType.UNKNOWN) fileType = FileTypes.GetFileType(extension); // If we still got unknown, just return null if (fileType == SupportedFileType.UNKNOWN) return null; #region Non-Archive File Types // Create a detectable for the given file type var detectable = Factory.CreateDetectable(fileType); // If we're scanning file contents if (detectable != null && ScanContents) { // If we have an executable, it needs to bypass normal handling if (detectable is Executable executable) { executable.IncludeGameEngines = ScanGameEngines; executable.IncludePackers = ScanPackers; var subProtections = ProcessExecutable(executable, fileName, stream); if (subProtections != null) AppendToDictionary(protections, subProtections); } // Otherwise, use the default implementation else { var subProtections = Handler.HandleDetectable(detectable, fileName, stream, IncludeDebug); if (subProtections != null) AppendToDictionary(protections, fileName, subProtections); } string subProtection = detectable.Detect(stream, fileName, IncludeDebug); if (!string.IsNullOrWhiteSpace(subProtection)) { // If we have an indicator of multiple protections if (subProtection.Contains(';')) { var splitProtections = subProtection.Split(';'); AppendToDictionary(protections, fileName, splitProtections); } else { AppendToDictionary(protections, fileName, subProtection); } } } #endregion #region Archive File Types // Create an extractable for the given file type var extractable = Factory.CreateExtractable(fileType); // If we're scanning archives if (extractable != null && ScanArchives) { var subProtections = Handler.HandleExtractable(extractable, fileName, stream, this); if (subProtections != null) AppendToDictionary(protections, subProtections); } #endregion } catch (Exception ex) { if (IncludeDebug) Console.WriteLine(ex); AppendToDictionary(protections, fileName, IncludeDebug ? ex.ToString() : "[Exception opening file, please try again]"); } // Clear out any empty keys ClearEmptyKeys(protections); return protections; } #endregion #region Executable Handling /// /// Process scanning for an Executable type /// /// Executable instance for processing /// Name of the source file of the stream, for tracking /// Stream to scan the contents of /// /// Ideally, we wouldn't need to circumvent the proper handling of file types just for Executable, /// but due to the complexity of scanning, this is not currently possible. /// private ConcurrentDictionary> ProcessExecutable(Executable executable, string fileName, Stream stream) { // Try to create a wrapper for the proper executable type var wrapper = WrapperFactory.CreateExecutableWrapper(stream); if (wrapper == null) return null; // Create the output dictionary var protections = new ConcurrentDictionary>(); // Only use generic content checks if we're in debug mode if (IncludeDebug) { var subProtections = executable.RunContentChecks(fileName, stream, IncludeDebug); if (subProtections != null) AppendToDictionary(protections, fileName, subProtections.Values.ToArray()); } if (wrapper is MSDOS mz) { var subProtections = executable.RunMSDOSExecutableChecks(fileName, stream, mz, IncludeDebug); if (subProtections == null) return protections; // Append the returned values AppendToDictionary(protections, fileName, subProtections.Values.ToArray()); // If we have any extractable packers var extractedProtections = HandleExtractableProtections(subProtections.Keys, fileName, stream); if (extractedProtections != null) AppendToDictionary(protections, extractedProtections); } else if (wrapper is LinearExecutable lex) { var subProtections = executable.RunLinearExecutableChecks(fileName, stream, lex, IncludeDebug); if (subProtections == null) return protections; // Append the returned values AppendToDictionary(protections, fileName, subProtections.Values.ToArray()); // If we have any extractable packers var extractedProtections = HandleExtractableProtections(subProtections.Keys, fileName, stream); if (extractedProtections != null) AppendToDictionary(protections, extractedProtections); } else if (wrapper is NewExecutable nex) { var subProtections = executable.RunNewExecutableChecks(fileName, stream, nex, IncludeDebug); if (subProtections == null) return protections; // Append the returned values AppendToDictionary(protections, fileName, subProtections.Values.ToArray()); // If we have any extractable packers var extractedProtections = HandleExtractableProtections(subProtections.Keys, fileName, stream); if (extractedProtections != null) AppendToDictionary(protections, extractedProtections); } else if (wrapper is PortableExecutable pex) { var subProtections = executable.RunPortableExecutableChecks(fileName, stream, pex, IncludeDebug); if (subProtections == null) return protections; // Append the returned values AppendToDictionary(protections, fileName, subProtections.Values.ToArray()); // If we have any extractable packers var extractedProtections = HandleExtractableProtections(subProtections.Keys, fileName, stream); if (extractedProtections != null) AppendToDictionary(protections, extractedProtections); } return protections; } /// /// Handle extractable protections, such as executable packers /// /// Set of classes returned from Exectuable scans /// Name of the source file of the stream, for tracking /// Stream to scan the contents of /// Set of protections found from extraction, null on error private ConcurrentDictionary> HandleExtractableProtections(IEnumerable classes, string fileName, Stream stream) { // If we have an invalid set of classes if (classes?.Any() != true) return null; // Create the output dictionary var protections = new ConcurrentDictionary>(); // If we have any extractable packers var extractables = classes.Where(c => c is IExtractable).Select(c => c as IExtractable); Parallel.ForEach(extractables, extractable => { // Get the protection for the class, if possible var extractedProtections = Handler.HandleExtractable(extractable, fileName, stream, this); if (extractedProtections != null) AppendToDictionary(protections, extractedProtections); }); return protections; } #endregion } }