using System; using System.Collections.Concurrent; using System.IO; using BurnOutSharp.Interfaces; using BurnOutSharp.Tools; using StormLibSharp; namespace BurnOutSharp.FileType { public class MPQ : IScannable { /// public bool ShouldScan(byte[] magic) { if (magic.StartsWith(new byte?[] { 0x4d, 0x50, 0x51, 0x1a })) return true; return false; } /// public ConcurrentDictionary> Scan(Scanner scanner, string file) { if (!File.Exists(file)) return null; using (var fs = File.OpenRead(file)) { return Scan(scanner, fs, file); } } // TODO: Add stream opening support /// public ConcurrentDictionary> Scan(Scanner scanner, Stream stream, string file) { // If the mpq file itself fails try { string tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); Directory.CreateDirectory(tempPath); using (MpqArchive mpqArchive = new MpqArchive(file, FileAccess.Read)) { // Try to open the listfile string listfile = null; MpqFileStream listStream = mpqArchive.OpenFile("(listfile)"); // If we can't read the listfile, we just return if (!listStream.CanRead) return null; // Read the listfile in for processing using (StreamReader sr = new StreamReader(listStream)) { listfile = sr.ReadToEnd(); } // Split the listfile by newlines string[] listfileLines = listfile.Replace("\r\n", "\n").Split('\n'); // Loop over each entry foreach (string sub in listfileLines) { // If an individual entry fails try { string tempFile = Path.Combine(tempPath, sub); Directory.CreateDirectory(Path.GetDirectoryName(tempFile)); mpqArchive.ExtractFile(sub, tempFile); } catch (Exception ex) { if (scanner.IncludeDebug) Console.WriteLine(ex); } } } // Collect and format all found protections var protections = scanner.GetProtections(tempPath); // If temp directory cleanup fails try { Directory.Delete(tempPath, true); } catch (Exception ex) { if (scanner.IncludeDebug) Console.WriteLine(ex); } // Remove temporary path references Utilities.StripFromKeys(protections, tempPath); return protections; } catch (Exception ex) { if (scanner.IncludeDebug) Console.WriteLine(ex); } return null; } // http://zezula.net/en/mpq/mpqformat.html #region TEMPORARY AREA FOR MPQ FORMAT /// /// MPQ (MoPaQ) is an archive format developed by Blizzard Entertainment, /// purposed for storing data files, images, sounds, music and videos for /// their games. The name MoPaQ comes from the author of the format, /// Mike O'Brien (Mike O'brien PaCK). /// internal class MoPaQArchive { #region Constants #region Header Sizes public const int HeaderVersion1Size = 0x20; public const int HeaderVersion2Size = 0x2C; public const int HeaderVersion3Size = 0x44; public const int HeaderVersion4Size = 0xD0; #endregion #region Signatures /// /// Human-readable signature /// public static readonly string SignatureString = $"MPQ{(char)0x1A}"; /// /// Signature as an unsigned Int32 value /// public const uint SignatureValue = 0x1A51504D; /// /// Signature as a byte array /// public static readonly byte[] SignatureBytes = new byte[] { 0x4D, 0x50, 0x51, 0x1A }; #endregion #endregion #region Properties // Data before archive, ignored /// /// MPQ User Data (optional) /// public MoPaQUserData UserData { get; private set; } /// /// MPQ Header (required) /// public MoPaQArchiveHeader ArchiveHeader { get; private set; } // Files (optional) // Special files (optional) /// /// HET Table (optional) /// public MoPaQHetTable HetTable { get; private set; } /// /// BET Table (optional) /// public MoPaQBetTable BetTable { get; private set; } /// /// Hash Table (optional) /// public MoPaQHashEntry[] HashTable { get; private set; } /// /// Block Table (optional) /// public MoPaQBlockEntry[] BlockTable { get; private set; } /// /// Hi-Block Table (optional) /// /// /// Since World of Warcraft - The Burning Crusade, Blizzard extended /// the MPQ format to support archives larger than 4GB. The hi-block /// table holds the higher 16-bits of the file position in the MPQ. /// Hi-block table is plain array of 16-bit values. This table is /// not encrypted. /// public short[] HiBlockTable { get; private set; } // Strong digital signature #endregion } /// /// MPQ User Data are optional, and is commonly used in custom maps for /// Starcraft II. If MPQ User Data header is present, it contains an offset, /// from where the MPQ header should be searched. /// internal class MoPaQUserData { #region Constants public const int Size = 0x10; /// /// Human-readable signature /// public static readonly string SignatureString = $"MPQ{(char)0x1B}"; /// /// Signature as an unsigned Int32 value /// public const uint SignatureValue = 0x1B51504D; /// /// Signature as a byte array /// public static readonly byte[] SignatureBytes = new byte[] { 0x4D, 0x50, 0x51, 0x1B }; #endregion #region Properties /// /// The user data signature /// /// public uint Signature { get; private set; } /// /// Maximum size of the user data /// public int UserDataSize { get; private set; } /// /// Offset of the MPQ header, relative to the beginning of this header /// public int HeaderOffset { get; private set; } /// /// Appears to be size of user data header (Starcraft II maps) /// public int UserDataHeadersize { get; private set; } // TODO: Does this area contain extra data that should be read in? #endregion } /// /// MoPaQ archive header /// internal class MoPaQArchiveHeader { #region V1 Properties /// /// The MPQ archive signature /// public uint Signature { get; private set; } /// /// Size of the archive header /// public int HeaderSize { get; private set; } /// /// Size of MPQ archive /// /// /// This field is deprecated in the Burning Crusade MoPaQ format, and the size of the archive /// is calculated as the size from the beginning of the archive to the end of the hash table, /// block table, or extended block table (whichever is largest). /// public int ArchiveSize { get; private set; } /// /// 0 = Format 1 (up to The Burning Crusade) /// 1 = Format 2 (The Burning Crusade and newer) /// 2 = Format 3 (WoW - Cataclysm beta or newer) /// 3 = Format 4 (WoW - Cataclysm beta or newer) /// public short FormatVersion { get; private set; } /// /// Power of two exponent specifying the number of 512-byte disk sectors in each logical sector /// in the archive. The size of each logical sector in the archive is 512 * 2 ^ BlockSize. /// public short BlockSize { get; private set; } /// /// Offset to the beginning of the hash table, relative to the beginning of the archive. /// public int HashTablePosition { get; private set; } /// /// Offset to the beginning of the block table, relative to the beginning of the archive. /// public int BlockTablePosition { get; private set; } /// /// Number of entries in the hash table. Must be a power of two, and must be less than 2^16 for /// the original MoPaQ format, or less than 2^20 for the Burning Crusade format. /// public int HashTableSize { get; private set; } /// /// Number of entries in the block table /// public int BlockTableSize { get; private set; } #endregion #region V2 Properties /// /// Offset to the beginning of array of 16-bit high parts of file offsets. /// public long HiBlockTablePosition { get; private set; } /// /// High 16 bits of the hash table offset for large archives. /// public short HashTablePositionHi { get; private set; } /// /// High 16 bits of the block table offset for large archives. /// public short BlockTablePositionHi { get; private set; } #endregion #region V3 Properties /// /// 64-bit version of the archive size /// public long ArchiveSizeLong { get; private set; } /// /// 64-bit position of the BET table /// public long BetTablePosition { get; private set; } /// /// 64-bit position of the HET table /// public long HetTablePosition { get; private set; } #endregion #region V4 Properties /// /// Compressed size of the hash table /// public long HashTableSizeLong { get; private set; } /// /// Compressed size of the block table /// public long BlockTableSizeLong { get; private set; } /// /// Compressed size of the hi-block table /// public long HiBlockTableSize { get; private set; } /// /// Compressed size of the HET block /// public long HetTableSize { get; private set; } /// /// Compressed size of the BET block /// public long BetTablesize { get; private set; } /// /// Size of raw data chunk to calculate MD5. /// /// MD5 of each data chunk follows the raw file data. public int RawChunkSize { get; private set; } // TODO: Is there a byte[] here of size RawChunkSize? /// /// MD5 of the block table before decryption /// public byte[] BlockTableMD5 { get; private set; } = new byte[0x10]; /// /// MD5 of the hash table before decryption /// public byte[] HashTableMD5 { get; private set; } = new byte[0x10]; /// /// MD5 of the hi-block table /// public byte[] HiBlockTableMD5 { get; private set; } = new byte[0x10]; /// /// MD5 of the BET table before decryption /// public byte[] BetTableMD5 { get; private set; } = new byte[0x10]; /// /// MD5 of the HET table before decryption /// public byte[] HetTableMD5 { get; private set; } = new byte[0x10]; /// /// MD5 of the MPQ header from signature to (including) HetTableMD5 /// public byte[] MpqHeaderMD5 { get; private set; } = new byte[0x10]; #endregion } /// /// The HET table is present if the HetTablePos64 member of MPQ header is /// set to nonzero. This table can fully replace hash table. Depending on /// MPQ size, the pair of HET&BET table can be more efficient than Hash&Block /// table. HET table can be encrypted and compressed. /// internal class MoPaQHetTable { #region Constants public const int Size = 0x44; /// /// Human-readable signature /// public static readonly string SignatureString = $"HET{(char)0x1A}"; /// /// Signature as an unsigned Int32 value /// public const uint SignatureValue = 0x1A544548; /// /// Signature as a byte array /// public static readonly byte[] SignatureBytes = new byte[] { 0x48, 0x45, 0x54, 0x1A }; #endregion // TODO: Extract this out and make in common between HET and BET #region Common Table Headers /// /// 'HET\x1A' /// public uint Signature { get; private set; } /// /// Version. Seems to be always 1 /// public int Version { get; private set; } /// /// Size of the contained table /// public int DataSize { get; private set; } #endregion #region Properties /// /// Size of the entire hash table, including the header (in bytes) /// public int TableSize { get; private set; } /// /// Maximum number of files in the MPQ /// public int MaxFileCount { get; private set; } /// /// Size of the hash table (in bytes) /// public int HashTableSize { get; private set; } /// /// Effective size of the hash entry (in bits) /// public int HashEntrySize { get; private set; } /// /// Total size of file index (in bits) /// public int TotalIndexSize { get; private set; } /// /// Extra bits in the file index /// public int IndexSizeExtra { get; private set; } /// /// Effective size of the file index (in bits) /// public int IndexSize { get; private set; } /// /// Size of the block index subtable (in bytes) /// public int BlockTableSize { get; private set; } /// /// HET hash table. Each entry is 8 bits. /// /// Size is derived from HashTableSize public byte[] HashTable { get; private set; } // TODO: Implement both of these on parse // Array of file indexes. Bit size of each entry is taken from dwTotalIndexSize. // Table size is taken from dwHashTableSize. #endregion } /// /// he BET table is present if the BetTablePos64 member of MPQ header is set /// to nonzero. BET table is a successor of classic block table, and can fully /// replace it. It is also supposed to be more effective. /// internal class MoPaQBetTable { #region Constants public const int Size = 0x88; /// /// Human-readable signature /// public static readonly string SignatureString = $"BET{(char)0x1A}"; /// /// Signature as an unsigned Int32 value /// public const uint SignatureValue = 0x1A544542; /// /// Signature as a byte array /// public static readonly byte[] SignatureBytes = new byte[] { 0x42, 0x45, 0x54, 0x1A }; #endregion // TODO: Extract this out and make in common between HET and BET #region Common Table Headers /// /// 'BET\x1A' /// public uint Signature { get; private set; } /// /// Version. Seems to be always 1 /// public int Version { get; private set; } /// /// Size of the contained table /// public int DataSize { get; private set; } #endregion #region Properties /// /// Size of the entire hash table, including the header (in bytes) /// public int TableSize { get; private set; } /// /// Number of files in the BET table /// public int FileCount { get; private set; } /// /// Unknown, set to 0x10 /// public int Unknown { get; private set; } /// /// Size of one table entry (in bits) /// public int TableEntrySize { get; private set; } /// /// Bit index of the file position (within the entry record) /// public int FilePositionBitIndex { get; private set; } /// /// Bit index of the file size (within the entry record) /// public int FileSizeBitIndex { get; private set; } /// /// Bit index of the compressed size (within the entry record) /// public int CompressedSizeBitIndex { get; private set; } /// /// Bit index of the flag index (within the entry record) /// public int FlagIndexBitIndex { get; private set; } /// /// Bit index of the ??? (within the entry record) /// public int UnknownBitIndex { get; private set; } /// /// Bit size of file position (in the entry record) /// public int FilePositionBitCount { get; private set; } /// /// Bit size of file size (in the entry record) /// public int FileSizeBitCount { get; private set; } /// /// Bit size of compressed file size (in the entry record) /// public int CompressedSizeBitCount { get; private set; } /// /// Bit size of flags index (in the entry record) /// public int FlagIndexBitCount { get; private set; } /// /// Bit size of ??? (in the entry record) /// public int UnknownBitCount { get; private set; } /// /// Total size of the BET hash /// public int TotalBetHashSize { get; private set; } /// /// Extra bits in the BET hash /// public int BetHashSizeExtra { get; private set; } /// /// Effective size of BET hash (in bits) /// public int BetHashSize { get; private set; } /// /// Size of BET hashes array, in bytes /// public int BetHashArraySize { get; private set; } /// /// Number of flags in the following array /// public int FlagCount { get; private set; } /// /// Followed by array of file flags. Each entry is 32-bit size and its meaning is the same like /// /// Size from public int[] FlagsArray { get; private set; } // File table. Size of each entry is taken from dwTableEntrySize. // Size of the table is (dwTableEntrySize * dwMaxFileCount), round up to 8. // Array of BET hashes. Table size is taken from dwMaxFileCount from HET table #endregion } /// /// Hash table is used for searching files by name. The file name is converted to /// two 32-bit hash values, which are then used for searching in the table. The size /// of the hash table must always be a power of two. Each entry in the hash table /// also contains file locale and offset into block table. Size of one entry of hash /// table is 16 bytes. /// internal class MoPaQHashEntry { #region Constants public const int Size = 0x10; #endregion #region Properties /// /// The hash of the full file name (part A) /// public uint NameHashPartA { get; private set; } /// /// The hash of the full file name (part B) /// public uint NameHashPartB { get; private set; } /// /// The language of the file. This is a Windows LANGID data type, and uses the same values. /// 0 indicates the default language (American English), or that the file is language-neutral. /// public MoPaQLocale Locale { get; private set; } /// /// The platform the file is used for. 0 indicates the default platform. /// No other values have been observed. /// public short Platform { get; private set; } /// /// If the hash table entry is valid, this is the index into the block table of the file. /// Otherwise, one of the following two values: /// - FFFFFFFFh: Hash table entry is empty, and has always been empty. /// Terminates searches for a given file. /// - FFFFFFFEh: Hash table entry is empty, but was valid at some point (a deleted file). /// Does not terminate searches for a given file. /// public uint BlockIndex { get; private set; } #endregion } internal enum MoPaQLocale : short { Neutral = 0, AmericanEnglish = 0, ChineseTaiwan = 0x404, Czech = 0x405, German = 0x407, English = 0x409, Spanish = 0x40A, French = 0x40C, Italian = 0x410, Japanese = 0x411, Korean = 0x412, Polish = 0x415, Portuguese = 0x416, Russian = 0x419, EnglishUK = 0x809, } /// /// Block table contains informations about file sizes and way of their storage within /// the archive. It also contains the position of file content in the archive. Size /// of block table entry is (like hash table entry). The block table is also encrypted. /// internal class MoPaQBlockEntry { #region Constants public const int Size = 0x10; #endregion #region Properties /// /// Offset of the beginning of the file data, relative to the beginning of the archive. /// public int FilePosition { get; private set; } /// /// Compressed file size /// public int CompressedSize { get; private set; } /// /// Size of uncompressed file /// public int UncompressedSize { get; private set; } /// /// Flags for the file. /// public MoPaQFileFlags Flags { get; private set; } #endregion } [Flags] internal enum MoPaQFileFlags : uint { /// /// File is compressed using PKWARE Data compression library /// MPQ_FILE_IMPLODE = 0x00000100, /// /// File is compressed using combination of compression methods /// MPQ_FILE_COMPRESS = 0x00000200, /// /// The file is encrypted /// MPQ_FILE_ENCRYPTED = 0x00010000, /// /// The decryption key for the file is altered according to the /// position of the file in the archive /// MPQ_FILE_FIX_KEY = 0x00020000, /// /// The file contains incremental patch for an existing file in base MPQ /// MPQ_FILE_PATCH_FILE = 0x00100000, /// /// Instead of being divided to 0x1000-bytes blocks, the file is stored /// as single unit /// MPQ_FILE_SINGLE_UNIT = 0x01000000, /// /// File is a deletion marker, indicating that the file no longer exists. /// This is used to allow patch archives to delete files present in /// lower-priority archives in the search chain. The file usually has /// length of 0 or 1 byte and its name is a hash /// MPQ_FILE_DELETE_MARKER = 0x02000000, /// /// File has checksums for each sector (explained in the File Data section). /// Ignored if file is not compressed or imploded. /// MPQ_FILE_SECTOR_CRC = 0x04000000, /// /// Set if file exists, reset when the file was deleted /// MPQ_FILE_EXISTS = 0x80000000, } /// /// This structure contains size of the patch, flags and also MD5 of the patch. /// internal class MoPaQPatchInfo { #region Properties /// /// Length of patch info header, in bytes /// public int Length { get; private set; } /// /// Flags. 0x80000000 = MD5 (?) /// public uint Flags { get; private set; } /// /// Uncompressed size of the patch file /// public int DataSize { get; private set; } /// /// MD5 of the entire patch file after decompression /// public byte[] MD5 { get; private set; } = new byte[0x10]; /// /// The sector offset table (variable length) /// public int[] SectorOffsetTable { get; private set; } #endregion } /// /// Each incremental patch file in a patch MPQ starts with a header. It is supposed /// to be a structure with variable length. /// internal class MoPaQPatchHeader { #region Constants #region Signatures #region Patch Header /// /// Human-readable signature /// public static readonly string PatchSignatureString = $"PTCH"; /// /// Signature as an unsigned Int32 value /// public const uint PatchSignatureValue = 0x48435450; /// /// Signature as a byte array /// public static readonly byte[] PatchSignatureBytes = new byte[] { 0x50, 0x54, 0x43, 0x48 }; #endregion #region MD5 Block /// /// Human-readable signature /// public static readonly string Md5SignatureString = $"MD5_"; /// /// Signature as an unsigned Int32 value /// public const uint Md5SignatureValue = 0x5F35444D; /// /// Signature as a byte array /// public static readonly byte[] Md5SignatureBytes = new byte[] { 0x4D, 0x44, 0x35, 0x5F }; #endregion #region XFRM Block /// /// Human-readable signature /// public static readonly string XFRMSignatureString = $"XFRM"; /// /// Signature as an unsigned Int32 value /// public const uint XFRMSignatureValue = 0x4D524658; /// /// Signature as a byte array /// public static readonly byte[] XFRMSignatureBytes = new byte[] { 0x58, 0x46, 0x52, 0x4D }; #endregion #region BSDIFF Patch Type /// /// Human-readable signature /// public static readonly string BSDIFF40SignatureString = $"BSDIFF40"; /// /// Signature as an unsigned Int64 value /// public const ulong BSDIFF40SignatureValue = 0x3034464649445342; /// /// Signature as a byte array /// public static readonly byte[] BSDIFF40SignatureBytes = new byte[] { 0x42, 0x53, 0x44, 0x49, 0x46, 0x46, 0x34, 0x30 }; #endregion #endregion #endregion #region Properties #region PATCH Header /// /// 'PTCH' /// public uint PatchSignature { get; private set; } /// /// Size of the entire patch (decompressed) /// public int SizeOfPatchData { get; private set; } /// /// Size of the file before patch /// public int SizeBeforePatch { get; private set; } /// /// Size of file after patch /// public int SizeAfterPatch { get; private set; } #endregion #region MD5 Block /// /// 'MD5_' /// public uint Md5Signature { get; private set; } /// /// Size of the MD5 block, including the signature and size itself /// public int Md5BlockSize { get; private set; } /// /// MD5 of the original (unpached) file /// public byte[] Md5BeforePatch { get; private set; } = new byte[0x10]; /// /// MD5 of the patched file /// public byte[] Md5AfterPatch { get; private set; } = new byte[0x10]; #endregion #region XFRM Block /// /// 'XFRM' /// public uint XfrmSignature { get; private set; } /// /// Size of the XFRM block, includes XFRM header and patch data /// public int XfrmBlockSize { get; private set; } /// /// Type of patch ('BSD0' or 'COPY') /// public MoPaQPatchType PatchType { get; private set; } #endregion #region Patch Data - BSD0 /// /// 'BSDIFF40' signature /// public ulong BsdiffSignature { get; private set; } /// /// Size of CTRL block (in bytes) /// public long CtrlBlockSize { get; private set; } /// /// Size of DATA block (in bytes) /// public long DataBlockSize { get; private set; } /// /// Size of file after applying the patch (in bytes) /// public long NewFileSize { get; private set; } // TODO: Fill rest of data from http://zezula.net/en/mpq/patchfiles.html // CTRL block // DATA block // EXTRA block #endregion #region Patch Data - COPY /// /// File data are simply replaced by the data in the patch. /// public byte[] PatchDataCopy { get; private set; } #endregion #endregion } internal enum MoPaQPatchType : uint { /// /// Blizzard-modified version of BSDIFF40 incremental patch /// BSD0 = 0x30445342, /// /// Unknown /// BSDP = 0x50445342, /// /// Plain replace /// COPY = 0x59504F43, /// /// Unknown /// COUP = 0x50554F43, /// /// Unknown /// CPOG = 0x474F5043, } #endregion } }