From e2e65bfbdf036d8e5050dd6a41c8295f9800eba8 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Tue, 5 Jul 2022 22:35:58 -0700 Subject: [PATCH] Add some unused structures for MPQ --- BurnOutSharp/FileType/MPQ.cs | 755 +++++++++++++++++++++++++++++++++++ 1 file changed, 755 insertions(+) diff --git a/BurnOutSharp/FileType/MPQ.cs b/BurnOutSharp/FileType/MPQ.cs index 1f370610..af4c68a3 100644 --- a/BurnOutSharp/FileType/MPQ.cs +++ b/BurnOutSharp/FileType/MPQ.cs @@ -101,5 +101,760 @@ namespace BurnOutSharp.FileType return null; } + + // http://zezula.net/en/mpq/mpqformat.html + #region TEMPORARY AREA FOR MPQ FORMAT + + /// + /// MPQ (MoPaQ) is an archive format developed by Blizzard Entertainment, + /// purposed for storing data files, images, sounds, music and videos for + /// their games. The name MoPaQ comes from the author of the format, + /// Mike O'Brien (Mike O'brien PaCK). + /// + internal class MoPaQArchive + { + #region Constants + + #region Header Sizes + + public const int HeaderVersion1Size = 0x20; + + public const int HeaderVersion2Size = 0x2C; + + public const int HeaderVersion3Size = 0x44; + + public const int HeaderVersion4Size = 0xD0; + + #endregion + + #region Signatures + + /// + /// Human-readable signature + /// + public static readonly string SignatureString = $"MPQ{(char)0x1A}"; + + /// + /// Signature as an unsigned Int32 value + /// + public const uint SignatureValue = 0x1A51504D; + + /// + /// Signature as a byte array + /// + public static readonly byte[] SignatureBytes = new byte[] { 0x4D, 0x50, 0x51, 0x1A }; + + #endregion + + #endregion + + #region Properties + + // Data before archive, ignored + + /// + /// MPQ User Data (optional) + /// + public MoPaQUserData UserData { get; private set; } + + /// + /// MPQ Header (required) + /// + public MoPaQArchiveHeader ArchiveHeader { get; private set; } + + // Files (optional) + // Special files (optional) + + /// + /// HET Table (optional) + /// + public MoPaQHetTable HetTable { get; private set; } + + /// + /// BET Table (optional) + /// + public MoPaQBetTable BetTable { get; private set; } + + /// + /// Hash Table (optional) + /// + public MoPaQHashEntry[] HashTable { get; private set; } + + /// + /// Block Table (optional) + /// + public MoPaQBlockEntry[] BlockTable { get; private set; } + + /// + /// Hi-Block Table (optional) + /// + /// + /// Since World of Warcraft - The Burning Crusade, Blizzard extended + /// the MPQ format to support archives larger than 4GB. The hi-block + /// table holds the higher 16-bits of the file position in the MPQ. + /// Hi-block table is plain array of 16-bit values. This table is + /// not encrypted. + /// + public short[] HiBlockTable { get; private set; } + + // Strong digital signature + + #endregion + } + + /// + /// MPQ User Data are optional, and is commonly used in custom maps for + /// Starcraft II. If MPQ User Data header is present, it contains an offset, + /// from where the MPQ header should be searched. + /// + internal class MoPaQUserData + { + #region Constants + + public const int Size = 0x10; + + /// + /// Human-readable signature + /// + public static readonly string SignatureString = $"MPQ{(char)0x1B}"; + + /// + /// Signature as an unsigned Int32 value + /// + public const uint SignatureValue = 0x1B51504D; + + /// + /// Signature as a byte array + /// + public static readonly byte[] SignatureBytes = new byte[] { 0x4D, 0x50, 0x51, 0x1B }; + + #endregion + + #region Properties + + /// + /// The user data signature + /// + /// + public uint Signature { get; private set; } + + /// + /// Maximum size of the user data + /// + public int UserDataSize { get; private set; } + + /// + /// Offset of the MPQ header, relative to the beginning of this header + /// + public int HeaderOffset { get; private set; } + + /// + /// Appears to be size of user data header (Starcraft II maps) + /// + public int UserDataHeadersize { get; private set; } + + // TODO: Does this area contain extra data that should be read in? + + #endregion + } + + /// + /// MoPaQ archive header + /// + internal class MoPaQArchiveHeader + { + #region V1 Properties + + /// + /// The MPQ archive signature + /// + public uint Signature { get; private set; } + + /// + /// Size of the archive header + /// + public int HeaderSize { get; private set; } + + /// + /// Size of MPQ archive + /// + /// + /// This field is deprecated in the Burning Crusade MoPaQ format, and the size of the archive + /// is calculated as the size from the beginning of the archive to the end of the hash table, + /// block table, or extended block table (whichever is largest). + /// + public int ArchiveSize { get; private set; } + + /// + /// 0 = Format 1 (up to The Burning Crusade) + /// 1 = Format 2 (The Burning Crusade and newer) + /// 2 = Format 3 (WoW - Cataclysm beta or newer) + /// 3 = Format 4 (WoW - Cataclysm beta or newer) + /// + public short FormatVersion { get; private set; } + + /// + /// Power of two exponent specifying the number of 512-byte disk sectors in each logical sector + /// in the archive. The size of each logical sector in the archive is 512 * 2 ^ BlockSize. + /// + public short BlockSize { get; private set; } + + /// + /// Offset to the beginning of the hash table, relative to the beginning of the archive. + /// + public int HashTablePosition { get; private set; } + + /// + /// Offset to the beginning of the block table, relative to the beginning of the archive. + /// + public int BlockTablePosition { get; private set; } + + /// + /// Number of entries in the hash table. Must be a power of two, and must be less than 2^16 for + /// the original MoPaQ format, or less than 2^20 for the Burning Crusade format. + /// + public int HashTableSize { get; private set; } + + /// + /// Number of entries in the block table + /// + public int BlockTableSize { get; private set; } + + #endregion + + #region V2 Properties + + /// + /// Offset to the beginning of array of 16-bit high parts of file offsets. + /// + public long HiBlockTablePosition { get; private set; } + + /// + /// High 16 bits of the hash table offset for large archives. + /// + public short HashTablePositionHi { get; private set; } + + /// + /// High 16 bits of the block table offset for large archives. + /// + public short BlockTablePositionHi { get; private set; } + + #endregion + + #region V3 Properties + + /// + /// 64-bit version of the archive size + /// + public long ArchiveSizeLong { get; private set; } + + /// + /// 64-bit position of the BET table + /// + public long BetTablePosition { get; private set; } + + /// + /// 64-bit position of the HET table + /// + public long HetTablePosition { get; private set; } + + #endregion + + #region V4 Properties + + /// + /// Compressed size of the hash table + /// + public long HashTableSizeLong { get; private set; } + + /// + /// Compressed size of the block table + /// + public long BlockTableSizeLong { get; private set; } + + /// + /// Compressed size of the hi-block table + /// + public long HiBlockTableSize { get; private set; } + + /// + /// Compressed size of the HET block + /// + public long HetTableSize { get; private set; } + + /// + /// Compressed size of the BET block + /// + public long BetTablesize { get; private set; } + + /// + /// Size of raw data chunk to calculate MD5. + /// + /// MD5 of each data chunk follows the raw file data. + public int RawChunkSize { get; private set; } + + // TODO: Is there a byte[] here of size RawChunkSize? + + /// + /// MD5 of the block table before decryption + /// + public byte[] BlockTableMD5 { get; private set; } = new byte[0x10]; + + /// + /// MD5 of the hash table before decryption + /// + public byte[] HashTableMD5 { get; private set; } = new byte[0x10]; + + /// + /// MD5 of the hi-block table + /// + public byte[] HiBlockTableMD5 { get; private set; } = new byte[0x10]; + + /// + /// MD5 of the BET table before decryption + /// + public byte[] BetTableMD5 { get; private set; } = new byte[0x10]; + + /// + /// MD5 of the HET table before decryption + /// + public byte[] HetTableMD5 { get; private set; } = new byte[0x10]; + + /// + /// MD5 of the MPQ header from signature to (including) HetTableMD5 + /// + public byte[] MpqHeaderMD5 { get; private set; } = new byte[0x10]; + + #endregion + } + + /// + /// The HET table is present if the HetTablePos64 member of MPQ header is + /// set to nonzero. This table can fully replace hash table. Depending on + /// MPQ size, the pair of HET&BET table can be more efficient than Hash&Block + /// table. HET table can be encrypted and compressed. + /// + internal class MoPaQHetTable + { + #region Constants + + public const int Size = 0x44; + + /// + /// Human-readable signature + /// + public static readonly string SignatureString = $"HET{(char)0x1A}"; + + /// + /// Signature as an unsigned Int32 value + /// + public const uint SignatureValue = 0x1A544548; + + /// + /// Signature as a byte array + /// + public static readonly byte[] SignatureBytes = new byte[] { 0x48, 0x45, 0x54, 0x1A }; + + #endregion + + // TODO: Extract this out and make in common between HET and BET + #region Common Table Headers + + /// + /// 'HET\x1A' + /// + public uint Signature { get; private set; } + + /// + /// Version. Seems to be always 1 + /// + public int Version { get; private set; } + + /// + /// Size of the contained table + /// + public int DataSize { get; private set; } + + #endregion + + #region Properties + + /// + /// Size of the entire hash table, including the header (in bytes) + /// + public int TableSize { get; private set; } + + /// + /// Maximum number of files in the MPQ + /// + public int MaxFileCount { get; private set; } + + /// + /// Size of the hash table (in bytes) + /// + public int HashTableSize { get; private set; } + + /// + /// Effective size of the hash entry (in bits) + /// + public int HashEntrySize { get; private set; } + + /// + /// Total size of file index (in bits) + /// + public int TotalIndexSize { get; private set; } + + /// + /// Extra bits in the file index + /// + public int IndexSizeExtra { get; private set; } + + /// + /// Effective size of the file index (in bits) + /// + public int IndexSize { get; private set; } + + /// + /// Size of the block index subtable (in bytes) + /// + public int BlockTableSize { get; private set; } + + /// + /// HET hash table. Each entry is 8 bits. + /// + /// Size is derived from HashTableSize + public byte[] HashTable { get; private set; } + + // TODO: Implement both of these on parse + // Array of file indexes. Bit size of each entry is taken from dwTotalIndexSize. + // Table size is taken from dwHashTableSize. + + #endregion + } + + /// + /// he BET table is present if the BetTablePos64 member of MPQ header is set + /// to nonzero. BET table is a successor of classic block table, and can fully + /// replace it. It is also supposed to be more effective. + /// + internal class MoPaQBetTable + { + #region Constants + + public const int Size = 0x88; + + /// + /// Human-readable signature + /// + public static readonly string SignatureString = $"BET{(char)0x1A}"; + + /// + /// Signature as an unsigned Int32 value + /// + public const uint SignatureValue = 0x1A544542; + + /// + /// Signature as a byte array + /// + public static readonly byte[] SignatureBytes = new byte[] { 0x42, 0x45, 0x54, 0x1A }; + + #endregion + + // TODO: Extract this out and make in common between HET and BET + #region Common Table Headers + + /// + /// 'BET\x1A' + /// + public uint Signature { get; private set; } + + /// + /// Version. Seems to be always 1 + /// + public int Version { get; private set; } + + /// + /// Size of the contained table + /// + public int DataSize { get; private set; } + + #endregion + + #region Properties + + /// + /// Size of the entire hash table, including the header (in bytes) + /// + public int TableSize { get; private set; } + + /// + /// Number of files in the BET table + /// + public int FileCount { get; private set; } + + /// + /// Unknown, set to 0x10 + /// + public int Unknown { get; private set; } + + /// + /// Size of one table entry (in bits) + /// + public int TableEntrySize { get; private set; } + + /// + /// Bit index of the file position (within the entry record) + /// + public int FilePositionBitIndex { get; private set; } + + /// + /// Bit index of the file size (within the entry record) + /// + public int FileSizeBitIndex { get; private set; } + + /// + /// Bit index of the compressed size (within the entry record) + /// + public int CompressedSizeBitIndex { get; private set; } + + /// + /// Bit index of the flag index (within the entry record) + /// + public int FlagIndexBitIndex { get; private set; } + + /// + /// Bit index of the ??? (within the entry record) + /// + public int UnknownBitIndex { get; private set; } + + /// + /// Bit size of file position (in the entry record) + /// + public int FilePositionBitCount { get; private set; } + + /// + /// Bit size of file size (in the entry record) + /// + public int FileSizeBitCount { get; private set; } + + /// + /// Bit size of compressed file size (in the entry record) + /// + public int CompressedSizeBitCount { get; private set; } + + /// + /// Bit size of flags index (in the entry record) + /// + public int FlagIndexBitCount { get; private set; } + + /// + /// Bit size of ??? (in the entry record) + /// + public int UnknownBitCount { get; private set; } + + /// + /// Total size of the BET hash + /// + public int TotalBetHashSize { get; private set; } + + /// + /// Extra bits in the BET hash + /// + public int BetHashSizeExtra { get; private set; } + + /// + /// Effective size of BET hash (in bits) + /// + public int BetHashSize { get; private set; } + + /// + /// Size of BET hashes array, in bytes + /// + public int BetHashArraySize { get; private set; } + + /// + /// Number of flags in the following array + /// + public int FlagCount { get; private set; } + + /// + /// Followed by array of file flags. Each entry is 32-bit size and its meaning is the same like + /// + /// Size from + public int[] FlagsArray { get; private set; } + + // File table. Size of each entry is taken from dwTableEntrySize. + // Size of the table is (dwTableEntrySize * dwMaxFileCount), round up to 8. + + // Array of BET hashes. Table size is taken from dwMaxFileCount from HET table + + #endregion + } + + /// + /// Hash table is used for searching files by name. The file name is converted to + /// two 32-bit hash values, which are then used for searching in the table. The size + /// of the hash table must always be a power of two. Each entry in the hash table + /// also contains file locale and offset into block table. Size of one entry of hash + /// table is 16 bytes. + /// + internal class MoPaQHashEntry + { + #region Constants + + public const int Size = 0x10; + + #endregion + + #region Properties + + /// + /// The hash of the full file name (part A) + /// + public uint NameHashPartA { get; private set; } + + /// + /// The hash of the full file name (part B) + /// + public uint NameHashPartB { get; private set; } + + /// + /// The language of the file. This is a Windows LANGID data type, and uses the same values. + /// 0 indicates the default language (American English), or that the file is language-neutral. + /// + public MoPaQLocale Locale { get; private set; } + + /// + /// The platform the file is used for. 0 indicates the default platform. + /// No other values have been observed. + /// + public short Platform { get; private set; } + + /// + /// If the hash table entry is valid, this is the index into the block table of the file. + /// Otherwise, one of the following two values: + /// - FFFFFFFFh: Hash table entry is empty, and has always been empty. + /// Terminates searches for a given file. + /// - FFFFFFFEh: Hash table entry is empty, but was valid at some point (a deleted file). + /// Does not terminate searches for a given file. + /// + public uint BlockIndex { get; private set; } + + #endregion + } + + internal enum MoPaQLocale : short + { + Neutral = 0, + AmericanEnglish = 0, + ChineseTaiwan = 0x404, + Czech = 0x405, + German = 0x407, + English = 0x409, + Spanish = 0x40A, + French = 0x40C, + Italian = 0x410, + Japanese = 0x411, + Korean = 0x412, + Polish = 0x415, + Portuguese = 0x416, + Russian = 0x419, + EnglishUK = 0x809, + } + + /// + /// lock table contains informations about file sizes and way of their storage within + /// the archive. It also contains the position of file content in the archive. Size + /// of block table entry is (like hash table entry). The block table is also encrypted. + /// + internal class MoPaQBlockEntry + { + #region Constants + + public const int Size = 0x10; + + #endregion + + #region Properties + + /// + /// Offset of the beginning of the file data, relative to the beginning of the archive. + /// + public int FilePosition { get; private set; } + + /// + /// Compressed file size + /// + public int CompressedSize { get; private set; } + + /// + /// Size of uncompressed file + /// + public int UncompressedSize { get; private set; } + + /// + /// Flags for the file. + /// + public MoPaQFileFlags Flags { get; private set; } + + #endregion + } + + [Flags] + internal enum MoPaQFileFlags : uint + { + /// + /// File is compressed using PKWARE Data compression library + /// + MPQ_FILE_IMPLODE = 0x00000100, + + /// + /// File is compressed using combination of compression methods + /// + MPQ_FILE_COMPRESS = 0x00000200, + + /// + /// The file is encrypted + /// + MPQ_FILE_ENCRYPTED = 0x00010000, + + /// + /// The decryption key for the file is altered according to the + /// position of the file in the archive + /// + MPQ_FILE_FIX_KEY = 0x00020000, + + /// + /// The file contains incremental patch for an existing file in base MPQ + /// + MPQ_FILE_PATCH_FILE = 0x00100000, + + /// + /// Instead of being divided to 0x1000-bytes blocks, the file is stored + /// as single unit + /// + MPQ_FILE_SINGLE_UNIT = 0x01000000, + + /// + /// File is a deletion marker, indicating that the file no longer exists. + /// This is used to allow patch archives to delete files present in + /// lower-priority archives in the search chain. The file usually has + /// length of 0 or 1 byte and its name is a hash + /// + MPQ_FILE_DELETE_MARKER = 0x02000000, + + /// + /// File has checksums for each sector (explained in the File Data section). + /// Ignored if file is not compressed or imploded. + /// + MPQ_FILE_SECTOR_CRC = 0x04000000, + + /// + /// Set if file exists, reset when the file was deleted + /// + MPQ_FILE_EXISTS = 0x80000000, + } + + #endregion } }