From 1663345195cdf4f6b37280e028c0ff01ba1d49f1 Mon Sep 17 00:00:00 2001 From: gmipf Date: Wed, 11 Nov 2026 17:18:23 +0100 Subject: [PATCH] Add a UDF reader and wrapper (#100) * Add a UDF reader Reads the Volume Recognition Sequence, the Anchor Volume Descriptor Pointer (sector 256, then the end of the volume), the Main or Reserve Volume Descriptor Sequence, the File Set Descriptor and the directory hierarchy. Type 1 and Metadata Partitions are mapped, Virtual and Sparable partitions are not. File Entries, Allocation Extent Descriptors and directories are read in ascending sector order with wrap-around, adjacent sectors merged into one request. FindFileEntry follows one path and reads only the directories on it. Co-Authored-By: Claude Opus 5.5 * Add a UDF wrapper Prints the volume and file set structures and the directory hierarchy, and extracts the files. CreateDiscImageWrapper tries UDF after ISO 9660 for a stream at offset 0, so discs with both file systems still open as ISO 9660. Co-Authored-By: Claude Opus 5.5 * Treat the stream position as the start of the UDF volume The UDF reader addresses logical sectors from the start of the stream, so the wrapper hands it a view that starts at the current position and keeps that offset, like the other wrappers. CreateDiscImageWrapper no longer skips UDF for a stream that is not at offset 0. Co-Authored-By: Claude Opus 5.5 * Address review comments on the UDF code - Move the descriptor tag check to UDFExtensions, with tests - Parse Volume Structure Descriptors in their own method - Use the existing extensions for numbers, the partition type identifier and printing entity identifiers - Use Array.Exists in IsSafeName - Make Extract non-virtual and remove the printing test - Use collection expressions and string interpolation Co-Authored-By: Claude Opus 5.5 * Parse File Identifier Descriptors in their own method Co-Authored-By: Claude Opus 5.5 * Add a TODO to use the CRC from Hashing Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../UDFExtensionsTests.cs | 274 ++++ SabreTools.Data.Extensions/UDFExtensions.cs | 417 ++++++ .../UDFImageBuilder.cs | 294 ++++ .../UDFTests.cs | 289 ++++ SabreTools.Serialization.Readers/UDF.cs | 1296 +++++++++++++++++ .../SabreTools.Wrappers.Test.csproj | 4 + SabreTools.Wrappers.Test/UDFTests.cs | 162 +++ SabreTools.Wrappers/UDF.Extraction.cs | 172 +++ SabreTools.Wrappers/UDF.Printing.cs | 296 ++++ SabreTools.Wrappers/UDF.cs | 112 ++ SabreTools.Wrappers/WrapperFactory.cs | 8 + 11 files changed, 3324 insertions(+) create mode 100644 SabreTools.Data.Extensions.Test/UDFExtensionsTests.cs create mode 100644 SabreTools.Data.Extensions/UDFExtensions.cs create mode 100644 SabreTools.Serialization.Readers.Test/UDFImageBuilder.cs create mode 100644 SabreTools.Serialization.Readers.Test/UDFTests.cs create mode 100644 SabreTools.Serialization.Readers/UDF.cs create mode 100644 SabreTools.Wrappers.Test/UDFTests.cs create mode 100644 SabreTools.Wrappers/UDF.Extraction.cs create mode 100644 SabreTools.Wrappers/UDF.Printing.cs create mode 100644 SabreTools.Wrappers/UDF.cs diff --git a/SabreTools.Data.Extensions.Test/UDFExtensionsTests.cs b/SabreTools.Data.Extensions.Test/UDFExtensionsTests.cs new file mode 100644 index 00000000..c52bc618 --- /dev/null +++ b/SabreTools.Data.Extensions.Test/UDFExtensionsTests.cs @@ -0,0 +1,274 @@ +using SabreTools.Data.Models.UDF; +using Xunit; + +namespace SabreTools.Data.Extensions.Test +{ + public class UDFExtensionsTests + { + #region DecodeDCharacters + + [Fact] + public void DecodeDCharacters_Null_Empty() + { + byte[]? data = null; + Assert.Equal(string.Empty, data.DecodeDCharacters()); + } + + [Fact] + public void DecodeDCharacters_8Bit_Decoded() + { + byte[] data = [8, 0x41, 0x62, 0xE4]; + Assert.Equal("Abä", data.DecodeDCharacters()); + } + + [Fact] + public void DecodeDCharacters_16Bit_Decoded() + { + byte[] data = [16, 0x00, 0x41, 0x30, 0x42]; + Assert.Equal("Aあ", data.DecodeDCharacters()); + } + + [Fact] + public void DecodeDCharacters_UnknownCompression_Null() + { + byte[] data = [1, 0x41]; + Assert.Null(data.DecodeDCharacters()); + } + + [Fact] + public void DecodeDString_UsesLengthByte() + { + byte[] data = [8, 0x41, 0x42, 0x43, 0x00, 3]; + Assert.Equal("AB", data.DecodeDString()); + } + + [Fact] + public void DecodeDString_ZeroLength_Empty() + { + byte[] data = [8, 0x41, 0x00, 0]; + Assert.Equal(string.Empty, data.DecodeDString()); + } + + #endregion + + #region TryGetSector + + [Fact] + public void TryGetSector_Type1_StartPlusBlock() + { + var volume = CreateVolume(new Type1PartitionMap { PartitionNumber = 0 }); + bool actual = volume.TryGetSector(UDFExtensions.GetKey(0, 10), out long sector); + Assert.True(actual); + Assert.Equal(1010, sector); + } + + [Fact] + public void TryGetSector_Type1_OutsidePartition_False() + { + var volume = CreateVolume(new Type1PartitionMap { PartitionNumber = 0 }); + bool actual = volume.TryGetSector(UDFExtensions.GetKey(0, 100), out _); + Assert.False(actual); + } + + [Fact] + public void TryGetSector_UnknownPartitionReference_False() + { + var volume = CreateVolume(new Type1PartitionMap { PartitionNumber = 0 }); + bool actual = volume.TryGetSector(UDFExtensions.GetKey(1, 0), out _); + Assert.False(actual); + } + + [Fact] + public void TryGetSector_Metadata_ThroughMetadataFile() + { + // Metadata File at block 5 of partition 0: blocks 0-1 of the metadata partition are blocks 20-21, + // blocks 2-3 are blocks 40-41 + var volume = CreateVolume( + new Type1PartitionMap { PartitionNumber = 0 }, + new MetadataPartitionMap { PartitionNumber = 0, MetadataFileLocation = 5, MetadataMirrorFileLocation = 6 }); + volume.FileEntries[UDFExtensions.GetKey(0, 5)] = CreateShortAdEntry((4096, 20), (4096, 40)); + + Assert.True(volume.TryGetSector(UDFExtensions.GetKey(1, 1), out long first)); + Assert.Equal(1021, first); + Assert.True(volume.TryGetSector(UDFExtensions.GetKey(1, 2), out long second)); + Assert.Equal(1040, second); + Assert.False(volume.TryGetSector(UDFExtensions.GetKey(1, 4), out _)); + } + + [Fact] + public void TryGetSector_Metadata_FallsBackToMirror() + { + var volume = CreateVolume( + new Type1PartitionMap { PartitionNumber = 0 }, + new MetadataPartitionMap { PartitionNumber = 0, MetadataFileLocation = 5, MetadataMirrorFileLocation = 6 }); + volume.FileEntries[UDFExtensions.GetKey(0, 6)] = CreateShortAdEntry((2048, 30)); + + Assert.True(volume.TryGetSector(UDFExtensions.GetKey(1, 0), out long sector)); + Assert.Equal(1030, sector); + } + + #endregion + + #region GetAllocationDescriptors + + [Fact] + public void GetAllocationDescriptors_Embedded_Empty() + { + var entry = new FileEntry(); + entry.ICBTag.Flags = (ushort)AllocationDescriptorType.EMBEDDED; + entry.AllocationDescriptors = [1, 2, 3]; + + var actual = new Volume().GetAllocationDescriptors(0, entry); + Assert.NotNull(actual); + Assert.Empty(actual); + } + + [Fact] + public void GetAllocationDescriptors_ZeroLengthTerminates() + { + var entry = CreateShortAdEntry((2048, 20), (0, 99), (2048, 40)); + + var actual = new Volume().GetAllocationDescriptors(UDFExtensions.GetKey(2, 7), entry); + Assert.NotNull(actual); + var ad = Assert.Single(actual); + Assert.Equal(20u, ad.ExtentLocation.LogicalBlockNumber); + Assert.Equal((ushort)2, ad.ExtentLocation.PartitionReferenceNumber); + } + + [Fact] + public void GetAllocationDescriptors_MissingAllocationExtent_NullWithKey() + { + var entry = CreateShortAdEntry((2048, 20), ((3u << 30) | 2048, 50)); + + var actual = new Volume().GetAllocationDescriptors(UDFExtensions.GetKey(0, 7), entry, out long missingKey); + Assert.Null(actual); + Assert.Equal(UDFExtensions.GetKey(0, 50), missingKey); + } + + [Fact] + public void GetAllocationDescriptors_AllocationExtentLoop_Null() + { + var volume = new Volume(); + var entry = CreateShortAdEntry(((3u << 30) | 2048, 50)); + volume.AllocationExtentDescriptors[UDFExtensions.GetKey(0, 50)] = new AllocationExtentDescriptor + { + AllocationDescriptors = CreateShortAdEntry(((3u << 30) | 2048, 50)).AllocationDescriptors, + }; + + var actual = volume.GetAllocationDescriptors(UDFExtensions.GetKey(0, 7), entry); + Assert.Null(actual); + } + + #endregion + + #region IsValidDescriptor + + [Theory] + [InlineData(2)] + [InlineData(3)] + public void IsValidDescriptor_Valid_True(byte version) + { + byte[] data = CreateDescriptor(version); + Assert.True(data.IsValidDescriptor(0, data.Length, 20)); + } + + [Fact] + public void IsValidDescriptor_WrongChecksum_False() + { + byte[] data = CreateDescriptor(2); + data[4]++; + Assert.False(data.IsValidDescriptor(0, data.Length, 20)); + } + + [Fact] + public void IsValidDescriptor_UnknownVersion_False() + { + byte[] data = CreateDescriptor(4); + Assert.False(data.IsValidDescriptor(0, data.Length, 20)); + } + + [Fact] + public void IsValidDescriptor_WrongCrc_False() + { + byte[] data = CreateDescriptor(2); + data[16] ^= 0x01; + Assert.False(data.IsValidDescriptor(0, data.Length, 20)); + } + + [Fact] + public void IsValidDescriptor_CrcPastAvailable_NotChecked() + { + byte[] data = CreateDescriptor(2); + data[16] ^= 0x01; + Assert.True(data.IsValidDescriptor(0, 16, 20)); + } + + [Fact] + public void IsValidDescriptor_WrongLocation_False() + { + byte[] data = CreateDescriptor(2); + Assert.False(data.IsValidDescriptor(0, data.Length, 21)); + } + + #endregion + + /// + /// Descriptor with tag location 20 whose 3 bytes after the tag have the CRC #3299 (ECMA-167 3/7.2.6) + /// + private static byte[] CreateDescriptor(byte version) + { + byte[] data = new byte[19]; + data[0] = 1; + data[2] = version; + data[8] = 0x99; + data[9] = 0x32; + data[10] = 3; + data[12] = 20; + data[16] = 0x70; + data[17] = 0x6A; + data[18] = 0x77; + for (int i = 0; i < 16; i++) + { + if (i != 4) + data[4] += data[i]; + } + + return data; + } + + /// + /// Volume with partition 0 at sector 1000 (length 50) and the given partition maps + /// + private static Volume CreateVolume(params PartitionMap[] maps) + { + return new Volume + { + VolumeDescriptorSequence = + [ + new PartitionDescriptor { PartitionNumber = 0, PartitionStartingLocation = 1000, PartitionLength = 50 }, + new LogicalVolumeDescriptor { LogicalBlockSize = 2048, PartitionMaps = maps }, + ], + }; + } + + /// + /// File Entry with the given short allocation descriptors (extent length, position) + /// + private static FileEntry CreateShortAdEntry(params (uint Length, uint Position)[] extents) + { + var data = new byte[extents.Length * 8]; + for (int i = 0; i < extents.Length; i++) + { + for (int b = 0; b < 4; b++) + { + data[(i * 8) + b] = (byte)(extents[i].Length >> (8 * b)); + data[(i * 8) + 4 + b] = (byte)(extents[i].Position >> (8 * b)); + } + } + + var entry = new FileEntry { AllocationDescriptors = data }; + entry.ICBTag.Flags = (ushort)AllocationDescriptorType.SHORT; + return entry; + } + } +} diff --git a/SabreTools.Data.Extensions/UDFExtensions.cs b/SabreTools.Data.Extensions/UDFExtensions.cs new file mode 100644 index 00000000..64c95e01 --- /dev/null +++ b/SabreTools.Data.Extensions/UDFExtensions.cs @@ -0,0 +1,417 @@ +using System.Collections.Generic; +using System.Text; +using SabreTools.Data.Models.UDF; +using SabreTools.Numerics.Extensions; + +namespace SabreTools.Data.Extensions +{ + public static class UDFExtensions + { + /// + /// Maximum number of Allocation Extent Descriptors followed for one File Entry + /// + private const int MaximumAllocationExtents = 1024; + + /// + /// CRC-ITU-T (CCITT, polynomial 0x1021) table for descriptor CRCs + /// + private static readonly ushort[] CrcTable = CreateCrcTable(); + + /// + /// Get the dictionary key of a logical block address + /// + /// Logical block address + /// ((long)PartitionReferenceNumber << 32) | LogicalBlockNumber + public static long GetKey(this LogicalBlockAddress address) + => GetKey(address.PartitionReferenceNumber, address.LogicalBlockNumber); + + /// + /// Get the dictionary key of a logical block address + /// + /// Index of the partition map + /// Logical block number within the partition + /// ((long)partitionReferenceNumber << 32) | logicalBlockNumber + public static long GetKey(ushort partitionReferenceNumber, uint logicalBlockNumber) + => ((long)partitionReferenceNumber << 32) | logicalBlockNumber; + + /// + /// Get the extent type from an allocation descriptor's extent length + /// + public static ExtentType GetExtentType(uint extentLength) + => (ExtentType)(extentLength >> Constants.ExtentTypeShift); + + /// + /// Get the extent length in bytes from an allocation descriptor's extent length + /// + public static uint GetExtentLength(uint extentLength) + => extentLength & Constants.ExtentLengthMask; + + /// + /// Get the type of the allocation descriptors of an ICB + /// + public static AllocationDescriptorType GetAllocationDescriptorType(this ICBTag tag) + => (AllocationDescriptorType)(tag.Flags & Constants.AllocationDescriptorTypeMask); + + /// + /// Get the prevailing Logical Volume Descriptor (highest Volume Descriptor Sequence Number) + /// + /// Volume to search + /// Logical Volume Descriptor on success, null if none was recorded + public static LogicalVolumeDescriptor? GetLogicalVolumeDescriptor(this Volume volume) + { + LogicalVolumeDescriptor? prevailing = null; + foreach (var descriptor in volume.VolumeDescriptorSequence) + { + if (descriptor is LogicalVolumeDescriptor lvd + && (prevailing is null || lvd.VolumeDescriptorSequenceNumber > prevailing.VolumeDescriptorSequenceNumber)) + { + prevailing = lvd; + } + } + + return prevailing; + } + + /// + /// Get the prevailing Partition Descriptor of a partition (highest Volume Descriptor Sequence Number) + /// + /// Volume to search + /// Partition number + /// Partition Descriptor on success, null if none was recorded + public static PartitionDescriptor? GetPartitionDescriptor(this Volume volume, ushort partitionNumber) + { + PartitionDescriptor? prevailing = null; + foreach (var descriptor in volume.VolumeDescriptorSequence) + { + if (descriptor is PartitionDescriptor pd + && pd.PartitionNumber == partitionNumber + && (prevailing is null || pd.VolumeDescriptorSequenceNumber > prevailing.VolumeDescriptorSequenceNumber)) + { + prevailing = pd; + } + } + + return prevailing; + } + + /// + /// Get the partition reference number of the Type 1 partition map for a partition number + /// + /// Logical Volume Descriptor holding the partition maps + /// Partition number + /// Partition reference number on success, -1 if there is no such map + public static int GetType1PartitionReference(this LogicalVolumeDescriptor lvd, ushort partitionNumber) + { + for (int i = 0; i < lvd.PartitionMaps.Length; i++) + { + if (lvd.PartitionMaps[i] is Type1PartitionMap map && map.PartitionNumber == partitionNumber) + return i; + } + + return -1; + } + + /// + /// Map a logical block of a partition to a logical sector of the volume + /// Supports Type 1 partitions and Metadata Partitions (the Metadata File's File Entry must be in FileEntries) + /// + /// Volume containing the Volume Descriptor Sequence + /// Logical block address key, see GetKey + /// Logical sector on success + /// True if the block could be mapped, false otherwise + public static bool TryGetSector(this Volume volume, long key, out long sector) + { + sector = -1; + + var lvd = volume.GetLogicalVolumeDescriptor(); + if (lvd is null) + return false; + + int partitionReference = (int)(key >> 32); + uint block = (uint)key; + if (partitionReference < 0 || partitionReference >= lvd.PartitionMaps.Length) + return false; + + var map = lvd.PartitionMaps[partitionReference]; + if (map is Type1PartitionMap type1) + { + var pd = volume.GetPartitionDescriptor(type1.PartitionNumber); + if (pd is null || block >= pd.PartitionLength) + return false; + + sector = (long)pd.PartitionStartingLocation + block; + return true; + } + else if (map is MetadataPartitionMap metadata) + { + // The Metadata File (or its mirror) is recorded in the Type 1 partition of the same partition number + int physical = lvd.GetType1PartitionReference(metadata.PartitionNumber); + if (physical < 0) + return false; + + if (!TryGetMetadataFileBlock(volume, lvd, GetKey((ushort)physical, metadata.MetadataFileLocation), block, out long physicalKey) + && !TryGetMetadataFileBlock(volume, lvd, GetKey((ushort)physical, metadata.MetadataMirrorFileLocation), block, out physicalKey)) + { + return false; + } + + return volume.TryGetSector(physicalKey, out sector); + } + + // Virtual and Sparable partitions are not supported + return false; + } + + /// + /// Map a logical block of a Metadata Partition through the Metadata File's allocation descriptors + /// + private static bool TryGetMetadataFileBlock(Volume volume, LogicalVolumeDescriptor lvd, long fileKey, uint block, out long physicalKey) + { + physicalKey = -1; + if (!volume.FileEntries.TryGetValue(fileKey, out var fileEntry)) + return false; + + var descriptors = volume.GetAllocationDescriptors(fileKey, fileEntry); + if (descriptors is null) + return false; + + uint blockSize = lvd.LogicalBlockSize; + long first = 0; + foreach (var ad in descriptors) + { + long blocks = (GetExtentLength(ad.ExtentLength) + blockSize - 1) / blockSize; + if (block < first + blocks) + { + if (GetExtentType(ad.ExtentLength) != ExtentType.RECORDED_AND_ALLOCATED) + return false; + + physicalKey = GetKey(ad.ExtentLocation.PartitionReferenceNumber, (uint)(ad.ExtentLocation.LogicalBlockNumber + (block - first))); + return true; + } + + first += blocks; + } + + return false; + } + + /// + /// Get the allocation descriptors of a File Entry, including those continued in Allocation Extent Descriptors, + /// as long allocation descriptors (short allocation descriptors get the partition of the File Entry) + /// + /// Volume containing the Allocation Extent Descriptors + /// Logical block address key of the File Entry + /// File Entry + /// Allocation descriptors (empty for embedded data), null if they are not supported or incomplete + public static List? GetAllocationDescriptors(this Volume volume, long key, FileEntry fileEntry) + => volume.GetAllocationDescriptors(key, fileEntry, out _); + + /// + /// Get the allocation descriptors of a File Entry, including those continued in Allocation Extent Descriptors, + /// as long allocation descriptors (short allocation descriptors get the partition of the File Entry) + /// + /// Volume containing the Allocation Extent Descriptors + /// Logical block address key of the File Entry + /// File Entry + /// Key of the first Allocation Extent Descriptor that is not in the volume, -1 if none + /// Allocation descriptors (empty for embedded data), null if they are not supported or incomplete + public static List? GetAllocationDescriptors(this Volume volume, long key, FileEntry fileEntry, out long missingKey) + { + missingKey = -1; + + var type = fileEntry.ICBTag.GetAllocationDescriptorType(); + if (type == AllocationDescriptorType.EMBEDDED) + return []; + + // Extended allocation descriptors are not allowed by UDF + if (type != AllocationDescriptorType.SHORT && type != AllocationDescriptorType.LONG) + return null; + + List result = []; + ushort partition = (ushort)(key >> 32); + byte[] descriptors = fileEntry.AllocationDescriptors; + int length = descriptors.Length; + + for (int extents = 0; extents <= MaximumAllocationExtents; extents++) + { + long nextKey = -1; + int size = type == AllocationDescriptorType.SHORT ? 8 : 16; + for (int offset = 0; offset + size <= length; offset += size) + { + var ad = new LongAllocationDescriptor(); + ad.ExtentLength = descriptors.ToUInt32LittleEndian(offset); + if (type == AllocationDescriptorType.SHORT) + { + ad.ExtentLocation.LogicalBlockNumber = descriptors.ToUInt32LittleEndian(offset + 4); + ad.ExtentLocation.PartitionReferenceNumber = partition; + } + else + { + ad.ExtentLocation.LogicalBlockNumber = descriptors.ToUInt32LittleEndian(offset + 4); + ad.ExtentLocation.PartitionReferenceNumber = descriptors.ToUInt16LittleEndian(offset + 8); + for (int i = 0; i < 6; i++) + { + ad.ImplementationUse[i] = descriptors[offset + 10 + i]; + } + } + + // A zero length terminates the allocation descriptors + if (GetExtentLength(ad.ExtentLength) == 0) + return result; + + if (GetExtentType(ad.ExtentLength) == ExtentType.NEXT_EXTENT) + { + nextKey = ad.ExtentLocation.GetKey(); + break; + } + + result.Add(ad); + } + + if (nextKey < 0) + return result; + + if (!volume.AllocationExtentDescriptors.TryGetValue(nextKey, out var aed)) + { + missingKey = nextKey; + return null; + } + + partition = (ushort)(nextKey >> 32); + descriptors = aed.AllocationDescriptors; + length = descriptors.Length; + } + + // Too many continuation extents, assume a loop + return null; + } + + /// + /// Check a descriptor tag: checksum, version, CRC (when it fits into the data) and location + /// + /// ECMA-167 3/7.2; receiving systems allow descriptor version 2 or 3 (3/7.2.2) + /// Data holding the descriptor + /// Offset of the descriptor + /// Number of bytes of the descriptor available from the offset + /// Expected tag location (logical sector or logical block number) + /// True if the tag is valid, false otherwise + public static bool IsValidDescriptor(this byte[] data, int offset, int available, uint location) + { + byte checksum = 0; + for (int i = 0; i < 16; i++) + { + if (i != 4) + checksum += data[offset + i]; + } + + if (checksum != data[offset + 4]) + return false; + + ushort version = data.ToUInt16LittleEndian(offset + 2); + if (version != 2 && version != 3) + return false; + + ushort crcLength = data.ToUInt16LittleEndian(offset + 10); + if (16 + crcLength <= available) + { + if (ComputeCrc(data, offset + 16, crcLength) != data.ToUInt16LittleEndian(offset + 8)) + return false; + } + + return data.ToUInt32LittleEndian(offset + 12) == location; + } + + /// + /// Decode OSTA Compressed Unicode d-characters (first byte = compression ID) + /// + /// Encoded characters + /// Decoded string, empty for no characters, null for an unknown compression ID + public static string? DecodeDCharacters(this byte[]? data) + => DecodeDCharacters(data, data?.Length ?? 0); + + /// + /// Decode an OSTA Compressed Unicode dstring (last byte = number of used bytes) + /// + /// Encoded dstring field + /// Decoded string, empty for no characters, null for an unknown compression ID + public static string? DecodeDString(this byte[]? data) + { + if (data is null || data.Length == 0) + return string.Empty; + + int length = data[data.Length - 1]; + if (length > data.Length - 1) + length = data.Length - 1; + + return DecodeDCharacters(data, length); + } + + /// + /// Decode the first length bytes of OSTA Compressed Unicode d-characters + /// + private static string? DecodeDCharacters(byte[]? data, int length) + { + if (data is null || length == 0) + return string.Empty; + + // 254 and 255 are the same encodings, used by UDF for deleted entries + byte compressionId = data[0]; + var builder = new StringBuilder(length); + if (compressionId == Constants.CompressionId8 || compressionId == 254) + { + for (int i = 1; i < length; i++) + { + builder.Append((char)data[i]); + } + } + else if (compressionId == Constants.CompressionId16 || compressionId == 255) + { + for (int i = 1; i + 1 < length; i += 2) + { + builder.Append((char)data.ToUInt16BigEndian(i)); + } + } + else + { + return null; + } + + return builder.ToString(); + } + + // TODO: Replace with CRC-16/XMODEM from Hashing once its CRC code is faster + /// + /// Compute the CRC-ITU-T of a range of bytes + /// + private static ushort ComputeCrc(byte[] data, int offset, int length) + { + ushort crc = 0; + for (int i = offset; i < offset + length; i++) + { + crc = (ushort)(CrcTable[((crc >> 8) ^ data[i]) & 0xFF] ^ (crc << 8)); + } + + return crc; + } + + /// + /// Create the CRC-ITU-T table + /// + private static ushort[] CreateCrcTable() + { + var table = new ushort[256]; + for (int i = 0; i < 256; i++) + { + ushort crc = (ushort)(i << 8); + for (int bit = 0; bit < 8; bit++) + { + crc = (crc & 0x8000) != 0 ? (ushort)((crc << 1) ^ 0x1021) : (ushort)(crc << 1); + } + + table[i] = crc; + } + + return table; + } + } +} diff --git a/SabreTools.Serialization.Readers.Test/UDFImageBuilder.cs b/SabreTools.Serialization.Readers.Test/UDFImageBuilder.cs new file mode 100644 index 00000000..c5460fff --- /dev/null +++ b/SabreTools.Serialization.Readers.Test/UDFImageBuilder.cs @@ -0,0 +1,294 @@ +using System; +using System.Text; + +namespace SabreTools.Serialization.Readers.Test +{ + /// + /// Builds a minimal UDF 1.02 image in memory (2048-byte sectors, one Type 1 partition at sector 260) + /// + /// + /// Partition blocks: + /// 0 File Set Descriptor + /// 2 root directory File Entry, contents in block 3: + /// parent, "README.TXT" (embedded data), "deleted.txt" (deleted), "Sub" (directory), + /// "Media" (last entry, no padding) + /// 4 "README.TXT" File Entry, data "hello world" embedded + /// 5 "Sub" File Entry, contents in block 6: parent, "data.bin" (16-bit name) + /// 7 "data.bin" File Entry, long_ad to block 8 (2048 bytes), then an Allocation Extent Descriptor + /// in block 10 with a long_ad to block 9 (952 bytes) + /// 8-9 "data.bin" data (byte i = i % 251) + /// 11 "Media" File Entry, one short_ad of 100 bytes allocated but not recorded + /// + internal static class UDFImageBuilder + { + public const int SectorSize = 2048; + public const int PartitionStart = 260; + public const int PartitionLength = 40; + public const int Sectors = 300; + public const int DataBinLength = 3000; + + /// + /// Build the image + /// + /// True to break the checksum of the root directory File Entry + /// True to change a byte of the root directory File Entry that only the CRC covers + /// True to record the partition map with length 0 and a map count of 3 + /// True to replace the Terminating Descriptor by a Volume Descriptor Pointer back to the start of the sequence + /// Name of the "Sub" directory in the root directory + /// True to add a directory "loop" to "Sub" that points back to the root directory + /// True to record "Media" in a partition block past the end of the image + public static byte[] Build(bool corruptRootEntry = false, + bool corruptRootEntryCrc = false, + bool emptyPartitionMap = false, + bool pointerLoop = false, + string subName = "Sub", + bool subLoop = false, + bool mediaPastEnd = false) + { + var image = new byte[Sectors * SectorSize]; + + // Volume Recognition Sequence + WriteStructure(image, 16, "BEA01"); + WriteStructure(image, 17, "NSR02"); + WriteStructure(image, 18, "TEA01"); + + // Anchor Volume Descriptor Pointer: main and reserve sequence at sector 32, 4 sectors + int avdp = 256 * SectorSize; + WriteUInt32(image, avdp + 16, 4 * SectorSize); + WriteUInt32(image, avdp + 20, 32); + WriteUInt32(image, avdp + 24, 4 * SectorSize); + WriteUInt32(image, avdp + 28, 32); + WriteTag(image, avdp, 2, 256, 512); + + // Primary Volume Descriptor + int pvd = 32 * SectorSize; + WriteDString(image, pvd + 24, 32, "TESTVOLUME"); + WriteTag(image, pvd, 1, 32, 512); + + // Partition Descriptor: partition 0 at sector 260 + int pd = 33 * SectorSize; + WriteUInt32(image, pd + 16, 1); + image[pd + 22] = 0; + WriteUInt32(image, pd + 188, PartitionStart); + WriteUInt32(image, pd + 192, mediaPastEnd ? PartitionLength + 20 : PartitionLength); + WriteTag(image, pd, 5, 33, 512); + + // Logical Volume Descriptor: block size, domain, File Set Descriptor at block 0, one Type 1 map + int lvd = 34 * SectorSize; + WriteUInt32(image, lvd + 16, 2); + WriteDString(image, lvd + 84, 128, "TESTVOLUME"); + WriteUInt32(image, lvd + 212, SectorSize); + WriteAscii(image, lvd + 217, "*OSTA UDF Compliant"); + image[lvd + 240] = 0x02; + image[lvd + 241] = 0x01; + WriteLongAd(image, lvd + 248, SectorSize, 0); + WriteUInt32(image, lvd + 264, 6); + WriteUInt32(image, lvd + 268, emptyPartitionMap ? 3 : 1); + image[lvd + 440] = 1; + image[lvd + 441] = (byte)(emptyPartitionMap ? 0 : 6); + image[lvd + 442] = 1; + WriteTag(image, lvd, 6, 34, 446); + + // Terminating Descriptor, or a Volume Descriptor Pointer back to the start of the sequence + int td = 35 * SectorSize; + if (pointerLoop) + { + WriteUInt32(image, td + 20, 4 * SectorSize); + WriteUInt32(image, td + 24, 32); + WriteTag(image, td, 3, 35, 512); + } + else + { + WriteTag(image, td, 8, 35, 512); + } + + // File Set Descriptor: root directory ICB at block 2 + int fsd = Block(0); + WriteDString(image, fsd + 304, 32, "TESTSET"); + WriteLongAd(image, fsd + 400, SectorSize, 2); + WriteTag(image, fsd, 256, 0, 512); + + // Root directory contents + int root = Block(3); + int length = 0; + length += WriteFid(image, root + length, 3, 0x0A, 2, []); + length += WriteFid(image, root + length, 3, 0x00, 4, Name8("README.TXT")); + length += WriteFid(image, root + length, 3, 0x04, 4, Name8("deleted.txt")); + length += WriteFid(image, root + length, 3, 0x02, 5, Name8(subName)); + length += WriteFid(image, root + length, 3, 0x00, 11, Name8("Media")); + WriteFileEntry(image, 2, 4, 0, length, ShortAd(length, 3)); + if (corruptRootEntry) + image[Block(2) + 4] ^= 0xFF; + if (corruptRootEntryCrc) + image[Block(2) + 36] ^= 0xFF; + + // "README.TXT", embedded + WriteFileEntry(image, 4, 5, 3, 11, Encoding.ASCII.GetBytes("hello world")); + + // "Sub" contents + int sub = Block(6); + length = 0; + length += WriteFid(image, sub + length, 6, 0x0A, 2, []); + length += WriteFid(image, sub + length, 6, 0x00, 7, Name16("data.bin")); + if (subLoop) + length += WriteFid(image, sub + length, 6, 0x02, 2, Name8("loop")); + + WriteFileEntry(image, 5, 4, 0, length, ShortAd(length, 6)); + + // "data.bin", long allocation descriptors continued in an Allocation Extent Descriptor + var ads = new byte[32]; + WriteLongAd(ads, 0, SectorSize, 8); + WriteLongAd(ads, 16, (3u << 30) | SectorSize, 10); + WriteFileEntry(image, 7, 5, 1, DataBinLength, ads); + for (int i = 0; i < DataBinLength; i++) + { + image[Block(8) + i] = (byte)(i % 251); + } + + int aed = Block(10); + WriteUInt32(image, aed + 20, 16); + WriteLongAd(image, aed + 24, DataBinLength - SectorSize, 9); + WriteTag(image, aed, 258, 10, 40); + + // "Media", allocated but not recorded, or recorded past the end of the image + if (mediaPastEnd) + WriteFileEntry(image, 11, 5, 0, 100, ShortAd(100, PartitionLength + 10)); + else + WriteFileEntry(image, 11, 5, 0, 100, ShortAd((1u << 30) | 100, 0)); + + return image; + } + + /// + /// Offset of a partition block in the image + /// + public static int Block(int block) => (PartitionStart + block) * SectorSize; + + private static void WriteStructure(byte[] image, int sector, string identifier) + { + int offset = sector * SectorSize; + image[offset] = 0; + WriteAscii(image, offset + 1, identifier); + image[offset + 6] = 1; + } + + private static int WriteFid(byte[] image, int offset, uint block, byte characteristics, uint icb, byte[] name) + { + image[offset + 16] = 1; + image[offset + 18] = characteristics; + image[offset + 19] = (byte)name.Length; + WriteLongAd(image, offset + 20, SectorSize, icb); + Array.Copy(name, 0, image, offset + 38, name.Length); + int length = (38 + name.Length + 3) & ~3; + WriteTag(image, offset, 257, block, length); + return length; + } + + private static void WriteFileEntry(byte[] image, int block, byte fileType, ushort adType, long informationLength, byte[] ads) + { + int offset = Block(block); + image[offset + 16 + 4] = 4; + image[offset + 16 + 11] = fileType; + image[offset + 16 + 18] = (byte)adType; + WriteUInt32(image, offset + 56, (uint)informationLength); + WriteUInt32(image, offset + 172, (uint)ads.Length); + Array.Copy(ads, 0, image, offset + 176, ads.Length); + WriteTag(image, offset, 261, (uint)block, 176 + ads.Length); + } + + private static byte[] ShortAd(uint length, uint position) + { + var ad = new byte[8]; + WriteUInt32(ad, 0, length); + WriteUInt32(ad, 4, position); + return ad; + } + + private static byte[] ShortAd(int length, uint position) => ShortAd((uint)length, position); + + private static void WriteLongAd(byte[] data, int offset, uint length, uint block) + { + WriteUInt32(data, offset, length); + WriteUInt32(data, offset + 4, block); + } + + private static void WriteLongAd(byte[] data, int offset, int length, uint block) + => WriteLongAd(data, offset, (uint)length, block); + + private static byte[] Name8(string name) + { + var bytes = new byte[name.Length + 1]; + bytes[0] = 8; + Encoding.ASCII.GetBytes(name, 0, name.Length, bytes, 1); + return bytes; + } + + private static byte[] Name16(string name) + { + var bytes = new byte[(name.Length * 2) + 1]; + bytes[0] = 16; + Encoding.BigEndianUnicode.GetBytes(name, 0, name.Length, bytes, 1); + return bytes; + } + + private static void WriteDString(byte[] image, int offset, int fieldLength, string value) + { + image[offset] = 8; + WriteAscii(image, offset + 1, value); + image[offset + fieldLength - 1] = (byte)(value.Length + 1); + } + + private static void WriteAscii(byte[] image, int offset, string value) + => Encoding.ASCII.GetBytes(value, 0, value.Length, image, offset); + + private static void WriteUInt32(byte[] data, int offset, uint value) + { + data[offset] = (byte)value; + data[offset + 1] = (byte)(value >> 8); + data[offset + 2] = (byte)(value >> 16); + data[offset + 3] = (byte)(value >> 24); + } + + private static void WriteUInt32(byte[] data, int offset, int value) => WriteUInt32(data, offset, (uint)value); + + /// + /// Write a descriptor tag covering the given descriptor length (version 2, CRC, checksum) + /// + private static void WriteTag(byte[] image, int offset, ushort identifier, uint location, int length) + { + image[offset] = (byte)identifier; + image[offset + 1] = (byte)(identifier >> 8); + image[offset + 2] = 2; + ushort crc = Crc(image, offset + 16, length - 16); + image[offset + 8] = (byte)crc; + image[offset + 9] = (byte)(crc >> 8); + image[offset + 10] = (byte)(length - 16); + image[offset + 11] = (byte)((length - 16) >> 8); + WriteUInt32(image, offset + 12, location); + + byte checksum = 0; + for (int i = 0; i < 16; i++) + { + if (i != 4) + checksum += image[offset + i]; + } + + image[offset + 4] = checksum; + } + + private static ushort Crc(byte[] data, int offset, int length) + { + ushort crc = 0; + for (int i = offset; i < offset + length; i++) + { + crc ^= (ushort)(data[i] << 8); + for (int bit = 0; bit < 8; bit++) + { + crc = (crc & 0x8000) != 0 ? (ushort)((crc << 1) ^ 0x1021) : (ushort)(crc << 1); + } + } + + return crc; + } + } +} diff --git a/SabreTools.Serialization.Readers.Test/UDFTests.cs b/SabreTools.Serialization.Readers.Test/UDFTests.cs new file mode 100644 index 00000000..ba07b5e5 --- /dev/null +++ b/SabreTools.Serialization.Readers.Test/UDFTests.cs @@ -0,0 +1,289 @@ +using System.IO; +using System.Linq; +using System.Text; +using SabreTools.Data.Extensions; +using SabreTools.Data.Models.UDF; +using Xunit; + +namespace SabreTools.Serialization.Readers.Test +{ + public class UDFTests + { + [Fact] + public void NullArray_Null() + { + byte[]? data = null; + int offset = 0; + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void EmptyArray_Null() + { + byte[]? data = []; + int offset = 0; + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void InvalidArray_Null() + { + byte[]? data = [.. Enumerable.Repeat(0xFF, 1024)]; + int offset = 0; + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void NullStream_Null() + { + Stream? data = null; + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + + [Fact] + public void EmptyStream_Null() + { + Stream? data = new MemoryStream([]); + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + + [Fact] + public void InvalidStream_Null() + { + Stream? data = new MemoryStream([.. Enumerable.Repeat(0xFF, 1024)]); + var deserializer = new UDF(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + + [Fact] + public void ValidImage_VolumeStructures() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build())); + + Assert.NotNull(volume); + Assert.Equal(3, volume.VolumeRecognitionSequence.Length); + Assert.NotNull(volume.AnchorVolumeDescriptorPointer); + Assert.Equal(4, volume.VolumeDescriptorSequence.Length); + var terminator = Assert.IsType(volume.VolumeDescriptorSequence[3]); + Assert.Equal(TagIdentifier.TERMINATING_DESCRIPTOR, terminator.DescriptorTag.TagIdentifier); + + var lvd = volume.GetLogicalVolumeDescriptor(); + Assert.NotNull(lvd); + Assert.Equal("TESTVOLUME", lvd.LogicalVolumeIdentifier.DecodeDString()); + Assert.IsType(Assert.Single(lvd.PartitionMaps)); + + Assert.NotNull(volume.FileSetDescriptor); + Assert.Equal("TESTSET", volume.FileSetDescriptor.FileSetIdentifier.DecodeDString()); + Assert.Equal(2u, volume.FileSetDescriptor.RootDirectoryICB.ExtentLocation.LogicalBlockNumber); + } + + [Fact] + public void ValidImage_DirectoryHierarchy() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build())); + + Assert.NotNull(volume); + + // Root, "README.TXT", "Sub", "data.bin", "Media"; the deleted entry is not followed + Assert.Equal(5, volume.FileEntries.Count); + Assert.Equal(2, volume.Directories.Count); + Assert.Single(volume.AllocationExtentDescriptors); + + // The last root entry has no padding and ends exactly at the end of the directory + var root = volume.Directories[UDFExtensions.GetKey(0, 2)]; + Assert.Equal(5, root.Length); + Assert.Equal("Media", root[4].FileIdentifier.DecodeDCharacters()); + Assert.Empty(root[4].Padding); + + var sub = volume.Directories[UDFExtensions.GetKey(0, 5)]; + Assert.Equal("data.bin", sub[1].FileIdentifier.DecodeDCharacters()); + } + + [Fact] + public void FindFileEntry_EmbeddedData() + { + var data = new MemoryStream(UDFImageBuilder.Build()); + var volume = new UDF().Deserialize(data); + Assert.NotNull(volume); + + var entry = UDF.FindFileEntry(data, volume, "README.TXT", out long key); + + Assert.NotNull(entry); + Assert.Equal(UDFExtensions.GetKey(0, 4), key); + Assert.Equal(AllocationDescriptorType.EMBEDDED, entry.ICBTag.GetAllocationDescriptorType()); + Assert.Equal("hello world", Encoding.ASCII.GetString(entry.AllocationDescriptors, 0, (int)entry.InformationLength)); + } + + [Fact] + public void FindFileEntry_CaseInsensitive() + { + var data = new MemoryStream(UDFImageBuilder.Build()); + var volume = new UDF().Deserialize(data); + Assert.NotNull(volume); + + var entry = UDF.FindFileEntry(data, volume, "/sub\\DATA.BIN", out long key); + + Assert.NotNull(entry); + Assert.Equal(UDFExtensions.GetKey(0, 7), key); + } + + [Fact] + public void FindFileEntry_DeletedOrMissing_Null() + { + var data = new MemoryStream(UDFImageBuilder.Build()); + var volume = new UDF().Deserialize(data); + Assert.NotNull(volume); + + Assert.Null(UDF.FindFileEntry(data, volume, "deleted.txt", out long deletedKey)); + Assert.Equal(-1, deletedKey); + Assert.Null(UDF.FindFileEntry(data, volume, "Sub/missing.bin", out _)); + Assert.Null(UDF.FindFileEntry(data, volume, "README.TXT/data.bin", out _)); + } + + [Fact] + public void FindFileEntry_WithoutDirectories_ReadsOnlyThePath() + { + var data = new MemoryStream(UDFImageBuilder.Build()); + var volume = new UDF().Deserialize(data, includeDirectories: false); + Assert.NotNull(volume); + Assert.Empty(volume.FileEntries); + Assert.Empty(volume.Directories); + + var entry = UDF.FindFileEntry(data, volume, "Sub/data.bin", out long key); + + Assert.NotNull(entry); + Assert.Equal(UDFExtensions.GetKey(0, 7), key); + + // Root, "Sub" and "data.bin" only + Assert.Equal(3, volume.FileEntries.Count); + Assert.Equal(2, volume.Directories.Count); + Assert.False(volume.FileEntries.ContainsKey(UDFExtensions.GetKey(0, 4))); + } + + [Fact] + public void AllocationExtent_Followed() + { + byte[] image = UDFImageBuilder.Build(); + var data = new MemoryStream(image); + var volume = new UDF().Deserialize(data); + Assert.NotNull(volume); + + long key = UDFExtensions.GetKey(0, 7); + var descriptors = volume.GetAllocationDescriptors(key, volume.FileEntries[key]); + + Assert.NotNull(descriptors); + Assert.Equal(2, descriptors.Count); + Assert.Equal(8u, descriptors[0].ExtentLocation.LogicalBlockNumber); + Assert.Equal(9u, descriptors[1].ExtentLocation.LogicalBlockNumber); + Assert.Equal((uint)UDFImageBuilder.DataBinLength, UDFExtensions.GetExtentLength(descriptors[0].ExtentLength) + UDFExtensions.GetExtentLength(descriptors[1].ExtentLength)); + + Assert.True(volume.TryGetSector(descriptors[1].ExtentLocation.GetKey(), out long sector)); + Assert.Equal(UDFImageBuilder.PartitionStart + 9, sector); + } + + [Fact] + public void NotRecordedExtent_Kept() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build())); + Assert.NotNull(volume); + + long key = UDFExtensions.GetKey(0, 11); + var descriptors = volume.GetAllocationDescriptors(key, volume.FileEntries[key]); + + Assert.NotNull(descriptors); + var ad = Assert.Single(descriptors); + Assert.Equal(ExtentType.NOT_RECORDED_BUT_ALLOCATED, UDFExtensions.GetExtentType(ad.ExtentLength)); + Assert.Equal(100u, UDFExtensions.GetExtentLength(ad.ExtentLength)); + } + + [Fact] + public void CorruptRootEntry_NoDirectories() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build(corruptRootEntry: true))); + + Assert.NotNull(volume); + Assert.NotNull(volume.FileSetDescriptor); + Assert.Empty(volume.FileEntries); + Assert.Empty(volume.Directories); + } + + [Fact] + public void CorruptRootEntryCrc_NoDirectories() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build(corruptRootEntryCrc: true))); + + Assert.NotNull(volume); + Assert.NotNull(volume.FileSetDescriptor); + Assert.Empty(volume.FileEntries); + Assert.Empty(volume.Directories); + } + + [Fact] + public void EmptyPartitionMap_EndsMapTable() + { + var data = new MemoryStream(UDFImageBuilder.Build(emptyPartitionMap: true)); + var anchor = UDF.ParseAnchorVolumeDescriptorPointer(data, Constants.AnchorVolumeDescriptorPointerSector); + Assert.NotNull(anchor); + + var sequence = UDF.ParseVolumeDescriptorSequence(data, anchor.MainVolumeDescriptorSequenceExtent); + + var lvd = Assert.Single(sequence.OfType()); + Assert.Empty(lvd.PartitionMaps); + } + + [Fact] + public void VolumeDescriptorPointerLoop_Stops() + { + var volume = new UDF().Deserialize(new MemoryStream(UDFImageBuilder.Build(pointerLoop: true))); + + Assert.NotNull(volume); + Assert.Equal(5, volume.FileEntries.Count); + } + + [Fact] + public void UnreadableSector_OtherEntriesRead() + { + // The "Sub" File Entry (block 5) cannot be read; "README.TXT" (block 4) is read in the same request + byte[] image = UDFImageBuilder.Build(); + long bad = UDFImageBuilder.Block(5); + var volume = new UDF().Deserialize(new ThrowingStream(image, bad, bad + UDFImageBuilder.SectorSize)); + + // Root, "README.TXT" and "Media" + long[] expected = [UDFExtensions.GetKey(0, 2), UDFExtensions.GetKey(0, 4), UDFExtensions.GetKey(0, 11)]; + Assert.NotNull(volume); + Assert.Equal(expected, volume.FileEntries.Keys.OrderBy(k => k)); + Assert.Single(volume.Directories); + } + + /// + /// Stream that throws an IOException for reads touching a range, like a drive with an unreadable sector + /// + private sealed class ThrowingStream(byte[] data, long badStart, long badEnd) : MemoryStream(data) + { + public override int Read(byte[] buffer, int offset, int count) + { + if (Position < badEnd && Position + count > badStart) + throw new IOException("Unreadable sector"); + + return base.Read(buffer, offset, count); + } + } + } +} diff --git a/SabreTools.Serialization.Readers/UDF.cs b/SabreTools.Serialization.Readers/UDF.cs new file mode 100644 index 00000000..8eae6405 --- /dev/null +++ b/SabreTools.Serialization.Readers/UDF.cs @@ -0,0 +1,1296 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SabreTools.Data.Extensions; +using SabreTools.Data.Models.UDF; +using SabreTools.IO.Extensions; +using SabreTools.Matching; +using SabreTools.Numerics.Extensions; +using SabreTools.Text.Extensions; + +#pragma warning disable IDE0017 // Simplify object initialization +namespace SabreTools.Serialization.Readers +{ + public class UDF : BaseBinaryReader + { + /// + /// Maximum number of Volume Descriptor Pointers followed in a Volume Descriptor Sequence + /// + private const int MaximumVolumeDescriptorPointers = 16; + + /// + public override Volume? Deserialize(Stream? data) + => Deserialize(data, includeDirectories: true); + + /// + /// + /// True to parse the whole directory hierarchy, false to parse only the volume and file set structures + /// (use FindFileEntry to look up single paths afterwards) + /// + /// Logical sectors are addressed from the start of the stream + public Volume? Deserialize(Stream? data, bool includeDirectories) + { + // If the data is invalid + if (data is null || !data.CanRead || !data.CanSeek) + return null; + + try + { + // Create a new Volume to fill + var volume = new Volume(); + + // Read the Volume Recognition Sequence + volume.VolumeRecognitionSequence = ParseVolumeRecognitionSequence(data); + + // Read the Anchor Volume Descriptor Pointer at sector 256, then at N - 256 and N (anchor points, ECMA-167 3/8.4.2.1) + var anchor = ParseAnchorVolumeDescriptorPointer(data, Constants.AnchorVolumeDescriptorPointerSector); + long lastSector = (data.Length / Constants.SectorSize) - 1; + if (anchor is null && lastSector > Constants.AnchorVolumeDescriptorPointerSector) + anchor = ParseAnchorVolumeDescriptorPointer(data, lastSector - Constants.AnchorVolumeDescriptorPointerSector); + if (anchor is null && lastSector > Constants.AnchorVolumeDescriptorPointerSector) + anchor = ParseAnchorVolumeDescriptorPointer(data, lastSector); + if (anchor is null) + return null; + + volume.AnchorVolumeDescriptorPointer = anchor; + + // Read the Main Volume Descriptor Sequence, then the equivalent Reserve one (ECMA-167 3/8.4.2.2) if the main one is unusable + volume.VolumeDescriptorSequence = ParseVolumeDescriptorSequence(data, anchor.MainVolumeDescriptorSequenceExtent); + if (!HasLogicalVolume(volume)) + volume.VolumeDescriptorSequence = ParseVolumeDescriptorSequence(data, anchor.ReserveVolumeDescriptorSequenceExtent); + if (!HasLogicalVolume(volume)) + return null; + + // Read the File Entries of Metadata Files, needed to map Metadata Partition blocks + var lvd = volume.GetLogicalVolumeDescriptor()!; + foreach (var map in lvd.PartitionMaps) + { + if (map is not MetadataPartitionMap metadata) + continue; + + int physical = lvd.GetType1PartitionReference(metadata.PartitionNumber); + if (physical < 0) + continue; + + var state = new HierarchyState(data, volume, includeDirectories: false); + state.QueueEntry(UDFExtensions.GetKey((ushort)physical, metadata.MetadataFileLocation)); + state.QueueEntry(UDFExtensions.GetKey((ushort)physical, metadata.MetadataMirrorFileLocation)); + state.Run(); + } + + // Read the File Set Descriptor + var fsd = ParseFileSetDescriptor(data, volume, lvd.LogicalVolumeContentsUse.ExtentLocation); + if (fsd is null) + return null; + + volume.FileSetDescriptor = fsd; + + // Read the directory hierarchy from the root directory + if (includeDirectories) + { + var state = new HierarchyState(data, volume, includeDirectories: true); + state.QueueEntry(fsd.RootDirectoryICB.ExtentLocation.GetKey()); + state.Run(); + } + + return volume; + } + catch + { + // Ignore the actual error + return null; + } + } + + #region Lookup + + /// + /// Find the File Entry of a path by walking the directories on the path only + /// + /// + /// Names are compared exactly first, then case-insensitively; deleted entries are ignored; + /// File Entries and directories read on the way are added to the volume (reused by later lookups) + /// + /// Stream containing the volume + /// Volume parsed by Deserialize (directories may be excluded) + /// Path from the root directory, separated by '/' or '\' + /// Logical block address key of the File Entry on success, -1 otherwise + /// File Entry on success, null if the path does not exist or could not be read + public static FileEntry? FindFileEntry(Stream data, Volume volume, string path, out long key) + { + key = -1; + if (volume.FileSetDescriptor is null) + return null; + + long current = volume.FileSetDescriptor.RootDirectoryICB.ExtentLocation.GetKey(); + var entry = LoadEntry(data, volume, current); + + string[] names = path.Split(['/', '\\'], StringSplitOptions.RemoveEmptyEntries); + foreach (string name in names) + { + if (entry is null || entry.ICBTag.FileType != FileType.DIRECTORY) + return null; + + if (!volume.Directories.TryGetValue(current, out var fids)) + return null; + + var fid = FindInDirectory(fids, name); + if (fid is null) + return null; + + current = fid.ICB.ExtentLocation.GetKey(); + entry = LoadEntry(data, volume, current); + } + + if (entry is null) + return null; + + key = current; + return entry; + } + + /// + /// Read a File Entry (with its Allocation Extent Descriptors, and its directory if it is one) unless already read + /// + private static FileEntry? LoadEntry(Stream data, Volume volume, long key) + { + if (!volume.FileEntries.TryGetValue(key, out var entry) + || (entry.ICBTag.FileType == FileType.DIRECTORY && !volume.Directories.ContainsKey(key))) + { + var state = new HierarchyState(data, volume, includeDirectories: false); + state.QueueEntry(key); + state.Run(); + volume.FileEntries.TryGetValue(key, out entry); + } + + return entry; + } + + /// + /// Find a name in the File Identifier Descriptors of a directory, exact match first, then case-insensitive + /// + private static FileIdentifierDescriptor? FindInDirectory(FileIdentifierDescriptor[] fids, string name) + { + FileIdentifierDescriptor? caseInsensitive = null; + foreach (var fid in fids) + { + if ((fid.FileCharacteristics & (FileCharacteristics.DELETED | FileCharacteristics.PARENT)) != 0) + continue; + + string? identifier = fid.FileIdentifier.DecodeDCharacters(); + if (identifier is null) + continue; + + if (string.Equals(identifier, name, StringComparison.Ordinal)) + return fid; + + if (caseInsensitive is null && string.Equals(identifier, name, StringComparison.OrdinalIgnoreCase)) + caseInsensitive = fid; + } + + return caseInsensitive; + } + + #endregion + + #region Volume Structure Parsing + + /// + /// Parse the Volume Recognition Sequence starting at sector 16 + /// + /// Stream to parse + /// Volume Structure Descriptors up to the first unknown one + public static VolumeStructureDescriptor[] ParseVolumeRecognitionSequence(Stream data) + { + List obj = []; + + long sector = Constants.VolumeRecognitionSequenceSector; + while (true) + { + byte[] block = ReadSectors(data, sector, 1); + + int offset = 0; + var vsd = ParseVolumeStructureDescriptor(block, ref offset); + + if (!IsKnownStandardIdentifier(vsd.StandardIdentifier)) + break; + + obj.Add(vsd); + sector++; + + // The extended area ends with the Terminating Extended Area Descriptor + if (vsd.StandardIdentifier.EqualsExactly(Constants.TerminatingExtendedAreaDescriptor)) + break; + } + + return [.. obj]; + } + + /// + /// Parse a Volume Structure Descriptor + /// + public static VolumeStructureDescriptor ParseVolumeStructureDescriptor(byte[] data, ref int offset) + { + var obj = new VolumeStructureDescriptor(); + + obj.StructureType = data.ReadByteValue(ref offset); + obj.StandardIdentifier = data.ReadBytes(ref offset, 5); + obj.StructureVersion = data.ReadByteValue(ref offset); + obj.StructureData = data.ReadBytes(ref offset, Constants.SectorSize - 7); + + return obj; + } + + /// + /// Parse an Anchor Volume Descriptor Pointer + /// + /// Stream to parse + /// Logical sector of the anchor + /// Filled AnchorVolumeDescriptorPointer on success, null on error + public static AnchorVolumeDescriptorPointer? ParseAnchorVolumeDescriptorPointer(Stream data, long sector) + { + byte[] block = ReadSectors(data, sector, 1); + if (!block.IsValidDescriptor(0, block.Length, (uint)sector)) + return null; + + int offset = 0; + var tag = ParseDescriptorTag(block, ref offset); + if (tag.TagIdentifier != TagIdentifier.ANCHOR_VOLUME_DESCRIPTOR_POINTER) + return null; + + var obj = new AnchorVolumeDescriptorPointer(); + obj.DescriptorTag = tag; + obj.MainVolumeDescriptorSequenceExtent = ParseExtentDescriptor(block, ref offset); + obj.ReserveVolumeDescriptorSequenceExtent = ParseExtentDescriptor(block, ref offset); + obj.Reserved = block.ReadBytes(ref offset, 480); + return obj; + } + + /// + /// Parse a Volume Descriptor Sequence, following Volume Descriptor Pointers + /// + /// Stream to parse + /// Extent of the sequence + /// Descriptors up to the Terminating Descriptor or the first invalid descriptor + public static Descriptor[] ParseVolumeDescriptorSequence(Stream data, ExtentDescriptor extent) + { + List obj = []; + + long sector = extent.ExtentLocation; + long end = sector + (extent.ExtentLength / Constants.SectorSize); + int pointers = 0; + while (sector < end) + { + byte[] block = ReadSectors(data, sector, 1); + if (!block.IsValidDescriptor(0, block.Length, (uint)sector)) + break; + + var descriptor = ParseVolumeDescriptor(block); + obj.Add(descriptor); + + TagIdentifier tagIdentifier = descriptor.DescriptorTag.TagIdentifier; + if (tagIdentifier == TagIdentifier.TERMINATING_DESCRIPTOR) + break; + + // Continue the sequence in the extent of a Volume Descriptor Pointer + if (descriptor is VolumeDescriptorPointer pointer) + { + if (++pointers > MaximumVolumeDescriptorPointers) + break; + + sector = pointer.NextVolumeDescriptorSequenceExtent.ExtentLocation; + end = sector + (pointer.NextVolumeDescriptorSequenceExtent.ExtentLength / Constants.SectorSize); + continue; + } + + sector++; + } + + return [.. obj]; + } + + /// + /// Parse a validated descriptor of the Volume Descriptor Sequence + /// + /// Logical sector holding the descriptor + /// Filled Descriptor + public static Descriptor ParseVolumeDescriptor(byte[] block) + { + int offset = 0; + var tag = ParseDescriptorTag(block, ref offset); + return tag.TagIdentifier switch + { + TagIdentifier.PRIMARY_VOLUME_DESCRIPTOR => ParsePrimaryVolumeDescriptor(block, ref offset, tag), + TagIdentifier.VOLUME_DESCRIPTOR_POINTER => ParseVolumeDescriptorPointer(block, ref offset, tag), + TagIdentifier.PARTITION_DESCRIPTOR => ParsePartitionDescriptor(block, ref offset, tag), + TagIdentifier.LOGICAL_VOLUME_DESCRIPTOR => ParseLogicalVolumeDescriptor(block, ref offset, tag), + + // Other descriptors are kept as raw data + TagIdentifier.ANCHOR_VOLUME_DESCRIPTOR_POINTER + or TagIdentifier.IMPLEMENTATION_USE_VOLUME_DESCRIPTOR + or TagIdentifier.UNALLOCATED_SPACE_DESCRIPTOR + or TagIdentifier.TERMINATING_DESCRIPTOR + or TagIdentifier.LOGICAL_VOLUME_INTEGRITY_DESCRIPTOR + or TagIdentifier.FILE_SET_DESCRIPTOR + or TagIdentifier.FILE_IDENTIFIER_DESCRIPTOR + or TagIdentifier.ALLOCATION_EXTENT_DESCRIPTOR + or TagIdentifier.INDIRECT_ENTRY + or TagIdentifier.TERMINAL_ENTRY + or TagIdentifier.FILE_ENTRY + or TagIdentifier.EXTENDED_ATTRIBUTE_HEADER_DESCRIPTOR + or TagIdentifier.UNALLOCATED_SPACE_ENTRY + or TagIdentifier.SPACE_BITMAP_DESCRIPTOR + or TagIdentifier.PARTITION_INTEGRITY_ENTRY + or TagIdentifier.EXTENDED_FILE_ENTRY => ParseGenericDescriptor(block, ref offset, tag), + _ => ParseGenericDescriptor(block, ref offset, tag), + }; + } + + /// + /// Parse a descriptor that is not parsed further after its tag + /// + private static GenericDescriptor ParseGenericDescriptor(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new GenericDescriptor(); + + obj.DescriptorTag = tag; + obj.Data = data.ReadBytes(ref offset, data.Length - offset); + + return obj; + } + + /// + /// Parse a Primary Volume Descriptor after its tag + /// + private static PrimaryVolumeDescriptor ParsePrimaryVolumeDescriptor(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new PrimaryVolumeDescriptor(); + + obj.DescriptorTag = tag; + obj.VolumeDescriptorSequenceNumber = data.ReadUInt32LittleEndian(ref offset); + obj.PrimaryVolumeDescriptorNumber = data.ReadUInt32LittleEndian(ref offset); + obj.VolumeIdentifier = data.ReadBytes(ref offset, 32); + obj.VolumeSequenceNumber = data.ReadUInt16LittleEndian(ref offset); + obj.MaximumVolumeSequenceNumber = data.ReadUInt16LittleEndian(ref offset); + obj.InterchangeLevel = data.ReadUInt16LittleEndian(ref offset); + obj.MaximumInterchangeLevel = data.ReadUInt16LittleEndian(ref offset); + obj.CharacterSetList = data.ReadUInt32LittleEndian(ref offset); + obj.MaximumCharacterSetList = data.ReadUInt32LittleEndian(ref offset); + obj.VolumeSetIdentifier = data.ReadBytes(ref offset, 128); + obj.DescriptorCharacterSet = ParseCharacterSetSpecification(data, ref offset); + obj.ExplanatoryCharacterSet = ParseCharacterSetSpecification(data, ref offset); + obj.VolumeAbstract = ParseExtentDescriptor(data, ref offset); + obj.VolumeCopyrightNotice = ParseExtentDescriptor(data, ref offset); + obj.ApplicationIdentifier = ParseEntityIdentifier(data, ref offset); + obj.RecordingDateAndTime = ParseTimestamp(data, ref offset); + obj.ImplementationIdentifier = ParseEntityIdentifier(data, ref offset); + obj.ImplementationUse = data.ReadBytes(ref offset, 64); + obj.PredecessorVolumeDescriptorSequenceLocation = data.ReadUInt32LittleEndian(ref offset); + obj.Flags = data.ReadUInt16LittleEndian(ref offset); + obj.Reserved = data.ReadBytes(ref offset, 22); + + return obj; + } + + /// + /// Parse a Volume Descriptor Pointer after its tag + /// + private static VolumeDescriptorPointer ParseVolumeDescriptorPointer(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new VolumeDescriptorPointer(); + + obj.DescriptorTag = tag; + obj.VolumeDescriptorSequenceNumber = data.ReadUInt32LittleEndian(ref offset); + obj.NextVolumeDescriptorSequenceExtent = ParseExtentDescriptor(data, ref offset); + obj.Reserved = data.ReadBytes(ref offset, 484); + + return obj; + } + + /// + /// Parse a Partition Descriptor after its tag + /// + private static PartitionDescriptor ParsePartitionDescriptor(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new PartitionDescriptor(); + + obj.DescriptorTag = tag; + obj.VolumeDescriptorSequenceNumber = data.ReadUInt32LittleEndian(ref offset); + obj.PartitionFlags = data.ReadUInt16LittleEndian(ref offset); + obj.PartitionNumber = data.ReadUInt16LittleEndian(ref offset); + obj.PartitionContents = ParseEntityIdentifier(data, ref offset); + obj.PartitionContentsUse = data.ReadBytes(ref offset, 128); + obj.AccessType = data.ReadUInt32LittleEndian(ref offset); + obj.PartitionStartingLocation = data.ReadUInt32LittleEndian(ref offset); + obj.PartitionLength = data.ReadUInt32LittleEndian(ref offset); + obj.ImplementationIdentifier = ParseEntityIdentifier(data, ref offset); + obj.ImplementationUse = data.ReadBytes(ref offset, 128); + obj.Reserved = data.ReadBytes(ref offset, 156); + + return obj; + } + + /// + /// Parse a Logical Volume Descriptor after its tag + /// + private static LogicalVolumeDescriptor ParseLogicalVolumeDescriptor(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new LogicalVolumeDescriptor(); + + obj.DescriptorTag = tag; + obj.VolumeDescriptorSequenceNumber = data.ReadUInt32LittleEndian(ref offset); + obj.DescriptorCharacterSet = ParseCharacterSetSpecification(data, ref offset); + obj.LogicalVolumeIdentifier = data.ReadBytes(ref offset, 128); + obj.LogicalBlockSize = data.ReadUInt32LittleEndian(ref offset); + obj.DomainIdentifier = ParseEntityIdentifier(data, ref offset); + obj.LogicalVolumeContentsUse = ParseLongAllocationDescriptor(data, ref offset); + obj.MapTableLength = data.ReadUInt32LittleEndian(ref offset); + obj.NumberOfPartitionMaps = data.ReadUInt32LittleEndian(ref offset); + obj.ImplementationIdentifier = ParseEntityIdentifier(data, ref offset); + obj.ImplementationUse = data.ReadBytes(ref offset, 128); + obj.IntegritySequenceExtent = ParseExtentDescriptor(data, ref offset); + + // Partition maps, bounded by the map table length and the logical sector + List maps = []; + int end = Math.Min(data.Length, offset + (int)Math.Min(obj.MapTableLength, (uint)data.Length)); + for (uint i = 0; i < obj.NumberOfPartitionMaps && offset + 2 <= end; i++) + { + int start = offset; + byte length = data[offset + 1]; + if (length < 2) + break; + + maps.Add(ParsePartitionMap(data, ref offset)); + offset = start + length; + } + + obj.PartitionMaps = [.. maps]; + + return obj; + } + + /// + /// Parse a Partition Map + /// + private static PartitionMap ParsePartitionMap(byte[] data, ref int offset) + { + int start = offset; + byte type = data[offset]; + byte length = data[offset + 1]; + + if (type == Constants.Type1PartitionMap && length >= 6) + { + var type1 = new Type1PartitionMap(); + type1.PartitionMapType = data.ReadByteValue(ref offset); + type1.PartitionMapLength = data.ReadByteValue(ref offset); + type1.VolumeSequenceNumber = data.ReadUInt16LittleEndian(ref offset); + type1.PartitionNumber = data.ReadUInt16LittleEndian(ref offset); + return type1; + } + + if (type == Constants.Type2PartitionMap && length >= 64) + { + offset += 4; + var identifier = ParseEntityIdentifier(data, ref offset); + offset = start; + + int identifierOffset = 0; + string? name = identifier.Identifier.ReadNullTerminatedAnsiString(ref identifierOffset); + Type2PartitionMap type2 = name == Constants.MetadataPartitionIdentifier + ? new MetadataPartitionMap() + : new Type2PartitionMap(); + + type2.PartitionMapType = data.ReadByteValue(ref offset); + type2.PartitionMapLength = data.ReadByteValue(ref offset); + type2.Reserved = data.ReadBytes(ref offset, 2); + type2.PartitionTypeIdentifier = ParseEntityIdentifier(data, ref offset); + type2.VolumeSequenceNumber = data.ReadUInt16LittleEndian(ref offset); + type2.PartitionNumber = data.ReadUInt16LittleEndian(ref offset); + if (type2 is MetadataPartitionMap metadata) + { + metadata.MetadataFileLocation = data.ReadUInt32LittleEndian(ref offset); + metadata.MetadataMirrorFileLocation = data.ReadUInt32LittleEndian(ref offset); + metadata.MetadataBitmapFileLocation = data.ReadUInt32LittleEndian(ref offset); + metadata.AllocationUnitSize = data.ReadUInt32LittleEndian(ref offset); + metadata.AlignmentUnitSize = data.ReadUInt16LittleEndian(ref offset); + metadata.Flags = data.ReadByteValue(ref offset); + metadata.Reserved2 = data.ReadBytes(ref offset, 5); + } + else + { + type2.PartitionTypeData = data.ReadBytes(ref offset, 24); + } + + return type2; + } + + // Unknown partition map type, keep the raw data + var unknown = new Type2PartitionMap(); + unknown.PartitionMapType = data.ReadByteValue(ref offset); + unknown.PartitionMapLength = data.ReadByteValue(ref offset); + unknown.PartitionTypeData = data.ReadBytes(ref offset, Math.Max(0, length - 2)); + return unknown; + } + + #endregion + + #region File Structure Parsing + + /// + /// Parse the File Set Descriptor + /// + /// Stream to parse + /// Volume containing the Volume Descriptor Sequence + /// Logical block address of the File Set Descriptor + /// Filled FileSetDescriptor on success, null on error + public static FileSetDescriptor? ParseFileSetDescriptor(Stream data, Volume volume, LogicalBlockAddress location) + { + if (!volume.TryGetSector(location.GetKey(), out long sector)) + return null; + + byte[] block = ReadSectors(data, sector, 1); + if (!block.IsValidDescriptor(0, block.Length, location.LogicalBlockNumber)) + return null; + + int offset = 0; + var tag = ParseDescriptorTag(block, ref offset); + if (tag.TagIdentifier != TagIdentifier.FILE_SET_DESCRIPTOR) + return null; + + var obj = new FileSetDescriptor(); + + obj.DescriptorTag = tag; + obj.RecordingDateAndTime = ParseTimestamp(block, ref offset); + obj.InterchangeLevel = block.ReadUInt16LittleEndian(ref offset); + obj.MaximumInterchangeLevel = block.ReadUInt16LittleEndian(ref offset); + obj.CharacterSetList = block.ReadUInt32LittleEndian(ref offset); + obj.MaximumCharacterSetList = block.ReadUInt32LittleEndian(ref offset); + obj.FileSetNumber = block.ReadUInt32LittleEndian(ref offset); + obj.FileSetDescriptorNumber = block.ReadUInt32LittleEndian(ref offset); + obj.LogicalVolumeIdentifierCharacterSet = ParseCharacterSetSpecification(block, ref offset); + obj.LogicalVolumeIdentifier = block.ReadBytes(ref offset, 128); + obj.FileSetCharacterSet = ParseCharacterSetSpecification(block, ref offset); + obj.FileSetIdentifier = block.ReadBytes(ref offset, 32); + obj.CopyrightFileIdentifier = block.ReadBytes(ref offset, 32); + obj.AbstractFileIdentifier = block.ReadBytes(ref offset, 32); + obj.RootDirectoryICB = ParseLongAllocationDescriptor(block, ref offset); + obj.DomainIdentifier = ParseEntityIdentifier(block, ref offset); + obj.NextExtent = ParseLongAllocationDescriptor(block, ref offset); + obj.SystemStreamDirectoryICB = ParseLongAllocationDescriptor(block, ref offset); + obj.Reserved = block.ReadBytes(ref offset, 32); + + return obj; + } + + /// + /// Parse a File Entry or Extended File Entry + /// + /// Logical block holding the entry + /// Logical block number the entry was read from + /// Filled FileEntry on success, null on error + public static FileEntry? ParseFileEntry(byte[] block, uint logicalBlockNumber) + { + if (!block.IsValidDescriptor(0, block.Length, logicalBlockNumber)) + return null; + + int offset = 0; + var tag = ParseDescriptorTag(block, ref offset); + bool extended = tag.TagIdentifier == TagIdentifier.EXTENDED_FILE_ENTRY; + if (!extended && tag.TagIdentifier != TagIdentifier.FILE_ENTRY) + return null; + + var obj = extended ? new ExtendedFileEntry() : new FileEntry(); + + obj.DescriptorTag = tag; + obj.ICBTag = ParseICBTag(block, ref offset); + obj.Uid = block.ReadUInt32LittleEndian(ref offset); + obj.Gid = block.ReadUInt32LittleEndian(ref offset); + obj.Permissions = block.ReadUInt32LittleEndian(ref offset); + obj.FileLinkCount = block.ReadUInt16LittleEndian(ref offset); + obj.RecordFormat = block.ReadByteValue(ref offset); + obj.RecordDisplayAttributes = block.ReadByteValue(ref offset); + obj.RecordLength = block.ReadUInt32LittleEndian(ref offset); + obj.InformationLength = block.ReadUInt64LittleEndian(ref offset); + if (obj is ExtendedFileEntry efe) + { + efe.ObjectSize = block.ReadUInt64LittleEndian(ref offset); + efe.LogicalBlocksRecorded = block.ReadUInt64LittleEndian(ref offset); + efe.AccessDateAndTime = ParseTimestamp(block, ref offset); + efe.ModificationDateAndTime = ParseTimestamp(block, ref offset); + efe.CreationDateAndTime = ParseTimestamp(block, ref offset); + efe.AttributeDateAndTime = ParseTimestamp(block, ref offset); + efe.Checkpoint = block.ReadUInt32LittleEndian(ref offset); + efe.Reserved = block.ReadUInt32LittleEndian(ref offset); + efe.ExtendedAttributeICB = ParseLongAllocationDescriptor(block, ref offset); + efe.StreamDirectoryICB = ParseLongAllocationDescriptor(block, ref offset); + } + else + { + obj.LogicalBlocksRecorded = block.ReadUInt64LittleEndian(ref offset); + obj.AccessDateAndTime = ParseTimestamp(block, ref offset); + obj.ModificationDateAndTime = ParseTimestamp(block, ref offset); + obj.AttributeDateAndTime = ParseTimestamp(block, ref offset); + obj.Checkpoint = block.ReadUInt32LittleEndian(ref offset); + obj.ExtendedAttributeICB = ParseLongAllocationDescriptor(block, ref offset); + } + + obj.ImplementationIdentifier = ParseEntityIdentifier(block, ref offset); + obj.UniqueID = block.ReadUInt64LittleEndian(ref offset); + obj.LengthOfExtendedAttributes = block.ReadUInt32LittleEndian(ref offset); + obj.LengthOfAllocationDescriptors = block.ReadUInt32LittleEndian(ref offset); + + // Both fields must fit into the logical block + if (obj.LengthOfExtendedAttributes > block.Length - offset + || obj.LengthOfAllocationDescriptors > block.Length - offset - obj.LengthOfExtendedAttributes) + { + return null; + } + + obj.ExtendedAttributes = block.ReadBytes(ref offset, (int)obj.LengthOfExtendedAttributes); + obj.AllocationDescriptors = block.ReadBytes(ref offset, (int)obj.LengthOfAllocationDescriptors); + + return obj; + } + + /// + /// Parse an Allocation Extent Descriptor + /// + /// Logical block holding the descriptor + /// Logical block number the descriptor was read from + /// Filled AllocationExtentDescriptor on success, null on error + public static AllocationExtentDescriptor? ParseAllocationExtentDescriptor(byte[] block, uint logicalBlockNumber) + { + if (!block.IsValidDescriptor(0, block.Length, logicalBlockNumber)) + return null; + + int offset = 0; + var tag = ParseDescriptorTag(block, ref offset); + if (tag.TagIdentifier != TagIdentifier.ALLOCATION_EXTENT_DESCRIPTOR) + return null; + + var obj = new AllocationExtentDescriptor(); + + obj.DescriptorTag = tag; + obj.PreviousAllocationExtentLocation = block.ReadUInt32LittleEndian(ref offset); + obj.LengthOfAllocationDescriptors = block.ReadUInt32LittleEndian(ref offset); + obj.AllocationDescriptors = block.ReadBytes(ref offset, (int)obj.LengthOfAllocationDescriptors); + + return obj; + } + + /// + /// Parse the File Identifier Descriptors of a directory + /// + /// Directory contents + /// Length of the directory contents in bytes + /// File Identifier Descriptors on success, null if the directory is invalid + public static FileIdentifierDescriptor[]? ParseDirectory(byte[] data, int length) + { + List obj = []; + + int offset = 0; + while (offset < length) + { + var tag = ParseDescriptorTag(data, ref offset); + if (tag.TagIdentifier != TagIdentifier.FILE_IDENTIFIER_DESCRIPTOR) + return null; + + var fid = ParseFileIdentifierDescriptor(data, ref offset, tag); + obj.Add(fid); + } + + return [.. obj]; + } + + /// + /// Parse a File Identifier Descriptor + /// + private static FileIdentifierDescriptor ParseFileIdentifierDescriptor(byte[] data, ref int offset, DescriptorTag tag) + { + var obj = new FileIdentifierDescriptor(); + + obj.DescriptorTag = tag; + obj.FileVersionNumber = data.ReadUInt16LittleEndian(ref offset); + obj.FileCharacteristics = (FileCharacteristics)data.ReadByteValue(ref offset); + obj.LengthOfFileIdentifier = data.ReadByteValue(ref offset); + obj.ICB = ParseLongAllocationDescriptor(data, ref offset); + obj.LengthOfImplementationUse = data.ReadUInt16LittleEndian(ref offset); + + // The descriptor is padded to 4 bytes + int used = 38 + obj.LengthOfImplementationUse + obj.LengthOfFileIdentifier; + int padded = (used + 3) & ~3; + obj.ImplementationUse = data.ReadBytes(ref offset, obj.LengthOfImplementationUse); + obj.FileIdentifier = data.ReadBytes(ref offset, obj.LengthOfFileIdentifier); + obj.Padding = data.ReadBytes(ref offset, padded - used); + + return obj; + } + + #endregion + + #region Common Structures + + /// + /// Parse a Descriptor Tag + /// + public static DescriptorTag ParseDescriptorTag(byte[] data, ref int offset) + { + var obj = new DescriptorTag(); + + obj.TagIdentifier = (TagIdentifier)data.ReadUInt16LittleEndian(ref offset); + obj.DescriptorVersion = data.ReadUInt16LittleEndian(ref offset); + obj.TagChecksum = data.ReadByteValue(ref offset); + obj.Reserved = data.ReadByteValue(ref offset); + obj.TagSerialNumber = data.ReadUInt16LittleEndian(ref offset); + obj.DescriptorCRC = data.ReadUInt16LittleEndian(ref offset); + obj.DescriptorCRCLength = data.ReadUInt16LittleEndian(ref offset); + obj.TagLocation = data.ReadUInt32LittleEndian(ref offset); + + return obj; + } + + /// + /// Parse an Extent Descriptor (extent_ad) + /// + public static ExtentDescriptor ParseExtentDescriptor(byte[] data, ref int offset) + { + var obj = new ExtentDescriptor(); + + obj.ExtentLength = data.ReadUInt32LittleEndian(ref offset); + obj.ExtentLocation = data.ReadUInt32LittleEndian(ref offset); + + return obj; + } + + /// + /// Parse a Long Allocation Descriptor (long_ad) + /// + public static LongAllocationDescriptor ParseLongAllocationDescriptor(byte[] data, ref int offset) + { + var obj = new LongAllocationDescriptor(); + + obj.ExtentLength = data.ReadUInt32LittleEndian(ref offset); + obj.ExtentLocation.LogicalBlockNumber = data.ReadUInt32LittleEndian(ref offset); + obj.ExtentLocation.PartitionReferenceNumber = data.ReadUInt16LittleEndian(ref offset); + obj.ImplementationUse = data.ReadBytes(ref offset, 6); + + return obj; + } + + /// + /// Parse an Entity Identifier (regid) + /// + public static EntityIdentifier ParseEntityIdentifier(byte[] data, ref int offset) + { + var obj = new EntityIdentifier(); + + obj.Flags = data.ReadByteValue(ref offset); + obj.Identifier = data.ReadBytes(ref offset, 23); + obj.IdentifierSuffix = data.ReadBytes(ref offset, 8); + + return obj; + } + + /// + /// Parse a Character Set Specification (charspec) + /// + public static CharacterSetSpecification ParseCharacterSetSpecification(byte[] data, ref int offset) + { + var obj = new CharacterSetSpecification(); + + obj.CharacterSetType = data.ReadByteValue(ref offset); + obj.CharacterSetInformation = data.ReadBytes(ref offset, 63); + + return obj; + } + + /// + /// Parse a Timestamp + /// + public static Timestamp ParseTimestamp(byte[] data, ref int offset) + { + var obj = new Timestamp(); + + obj.TypeAndTimezone = data.ReadUInt16LittleEndian(ref offset); + obj.Year = data.ReadInt16LittleEndian(ref offset); + obj.Month = data.ReadByteValue(ref offset); + obj.Day = data.ReadByteValue(ref offset); + obj.Hour = data.ReadByteValue(ref offset); + obj.Minute = data.ReadByteValue(ref offset); + obj.Second = data.ReadByteValue(ref offset); + obj.Centiseconds = data.ReadByteValue(ref offset); + obj.HundredsOfMicroseconds = data.ReadByteValue(ref offset); + obj.Microseconds = data.ReadByteValue(ref offset); + + return obj; + } + + /// + /// Parse an ICB Tag (icbtag) + /// + public static ICBTag ParseICBTag(byte[] data, ref int offset) + { + var obj = new ICBTag(); + + obj.PriorRecordedNumberOfDirectEntries = data.ReadUInt32LittleEndian(ref offset); + obj.StrategyType = data.ReadUInt16LittleEndian(ref offset); + obj.StrategyParameter = data.ReadBytes(ref offset, 2); + obj.MaximumNumberOfEntries = data.ReadUInt16LittleEndian(ref offset); + obj.Reserved = data.ReadByteValue(ref offset); + obj.FileType = (FileType)data.ReadByteValue(ref offset); + obj.ParentICBLocation.LogicalBlockNumber = data.ReadUInt32LittleEndian(ref offset); + obj.ParentICBLocation.PartitionReferenceNumber = data.ReadUInt16LittleEndian(ref offset); + obj.Flags = data.ReadUInt16LittleEndian(ref offset); + + return obj; + } + + #endregion + + #region Helpers + + /// + /// Check whether the volume has a Logical Volume Descriptor with a supported logical block size + /// + private static bool HasLogicalVolume(Volume volume) + { + var lvd = volume.GetLogicalVolumeDescriptor(); + return lvd is not null && lvd.LogicalBlockSize == Constants.SectorSize; + } + + /// + /// Check whether a standard identifier belongs to the Volume Recognition Sequence + /// + private static bool IsKnownStandardIdentifier(byte[] identifier) + { + return identifier.EqualsExactly(Constants.BeginningExtendedAreaDescriptor) + || identifier.EqualsExactly(Constants.NSR02) + || identifier.EqualsExactly(Constants.NSR03) + || identifier.EqualsExactly(Constants.TerminatingExtendedAreaDescriptor) + || identifier.EqualsExactly(Constants.CD001) + || identifier.EqualsExactly(Constants.BOOT2) + || identifier.EqualsExactly(Constants.CDW02); + } + + /// + /// Read whole logical sectors + /// + private static byte[] ReadSectors(Stream data, long sector, int count) + { + data.SeekIfPossible(sector * Constants.SectorSize, SeekOrigin.Begin); + return data.ReadBytes(count * Constants.SectorSize); + } + + #endregion + + #region Directory Hierarchy + + /// + /// Kind of a pending read + /// + private enum ReadKind + { + /// + /// File Entry or Extended File Entry + /// + Entry, + + /// + /// Allocation Extent Descriptor of a File Entry + /// + AllocationExtent, + + /// + /// Part of the contents of a directory + /// + DirectoryData, + } + + /// + /// Contents of a directory being assembled from its extents + /// + private sealed class DirectoryContents + { + public long Key; + public byte[] Data = []; + public int Pending; + } + + /// + /// A read waiting for the elevator + /// + private sealed class PendingRead + { + public long Sector; + public int Count; + public ReadKind Kind; + public long Key; + public DirectoryContents? Directory; + public int DirectoryOffset; + public int Length; + } + + /// + /// Orders pending reads by sector + /// + private sealed class PendingReadComparer : IComparer + { + public static readonly PendingReadComparer Instance = new(); + + public int Compare(PendingRead? x, PendingRead? y) + => (x?.Sector ?? 0).CompareTo(y?.Sector ?? 0); + } + + /// + /// Reads File Entries, Allocation Extent Descriptors and directories in ascending sector order (C-SCAN), + /// merging adjacent sectors into one request; structures found in the last request are served from it + /// + private sealed class HierarchyState + { + private readonly Stream _data; + private readonly Volume _volume; + private readonly bool _includeDirectories; + + /// + /// Pending reads ordered by sector + /// + private readonly List _pending = []; + + /// + /// Reads served from the last request, processed before the next request + /// + private readonly Queue> _ready = new(); + + /// + /// Keys of Allocation Extent Descriptors already queued, with the File Entries waiting for them + /// + private readonly Dictionary> _queuedExtents = []; + + /// + /// Sector after the last request + /// + private long _head = 0; + + /// + /// Data and first sector of the last request + /// + private byte[] _lastRun = []; + private long _lastRunSector = -1; + + /// Stream containing the volume + /// Volume to fill + /// True to read the entries of every directory read + public HierarchyState(Stream data, Volume volume, bool includeDirectories) + { + _data = data; + _volume = volume; + _includeDirectories = includeDirectories; + } + + /// + /// Queue a File Entry, or resolve it if it was already read + /// + public void QueueEntry(long key) + { + if (_volume.FileEntries.TryGetValue(key, out var entry)) + { + ResolveEntry(key, entry); + return; + } + + if (_volume.TryGetSector(key, out long sector)) + Queue(new PendingRead { Sector = sector, Count = 1, Kind = ReadKind.Entry, Key = key }); + } + + /// + /// Process pending reads until none are left + /// + public void Run() + { + while (true) + { + while (_ready.Count > 0) + { + var ready = _ready.Dequeue(); + Process(ready.Key, ready.Value); + } + + if (_pending.Count == 0) + return; + + // Next pending read at or after the head, wrapping around to the lowest sector + int first = FirstAtOrAfter(_head); + if (first >= _pending.Count) + first = 0; + + // Merge adjacent or overlapping reads into one request + long runStart = _pending[first].Sector; + long runEnd = runStart + _pending[first].Count; + int last = first; + while (last + 1 < _pending.Count && _pending[last + 1].Sector <= runEnd) + { + last++; + runEnd = Math.Max(runEnd, _pending[last].Sector + _pending[last].Count); + } + + var run = _pending.GetRange(first, last - first + 1); + _pending.RemoveRange(first, last - first + 1); + _head = runEnd; + + byte[]? buffer = null; + try + { + buffer = ReadSectors(_data, runStart, (int)(runEnd - runStart)); + } + catch (IOException) + { + // Unreadable sectors: retry each read alone below + } + + if (buffer is not null) + { + _lastRun = buffer; + _lastRunSector = runStart; + } + + foreach (var read in run) + { + byte[]? sectors = buffer is not null + ? Slice(buffer, (int)(read.Sector - runStart) * Constants.SectorSize, read.Count * Constants.SectorSize) + : TryReadSectors(read.Sector, read.Count); + if (sectors is not null) + Process(read, sectors); + } + } + } + + /// + /// Add a read to the pending reads, or to the ready reads if the last request contains it + /// + private void Queue(PendingRead read) + { + long runEnd = _lastRunSector + (_lastRun.Length / Constants.SectorSize); + if (_lastRunSector >= 0 && read.Sector >= _lastRunSector && read.Sector + read.Count <= runEnd) + { + int offset = (int)(read.Sector - _lastRunSector) * Constants.SectorSize; + _ready.Enqueue(new KeyValuePair(read, Slice(_lastRun, offset, read.Count * Constants.SectorSize))); + return; + } + + int index = _pending.BinarySearch(read, PendingReadComparer.Instance); + if (index < 0) + index = ~index; + + // Keep reads of the same sector in queue order + while (index < _pending.Count && _pending[index].Sector == read.Sector) + { + index++; + } + + _pending.Insert(index, read); + } + + /// + /// Index of the first pending read at or after a sector + /// + private int FirstAtOrAfter(long sector) + { + int low = 0, high = _pending.Count; + while (low < high) + { + int middle = low + ((high - low) / 2); + if (_pending[middle].Sector < sector) + low = middle + 1; + else + high = middle; + } + + return low; + } + + /// + /// Process the data of a read + /// + private void Process(PendingRead read, byte[] sectors) + { + switch (read.Kind) + { + case ReadKind.Entry: + var entry = ParseFileEntry(sectors, (uint)read.Key); + if (entry is null) + return; + + _volume.FileEntries[read.Key] = entry; + ResolveEntry(read.Key, entry); + break; + + case ReadKind.AllocationExtent: + var aed = ParseAllocationExtentDescriptor(sectors, (uint)read.Key); + if (aed is null) + return; + + _volume.AllocationExtentDescriptors[read.Key] = aed; + if (_queuedExtents.TryGetValue(read.Key, out var owners)) + { + foreach (long owner in owners) + { + if (_volume.FileEntries.TryGetValue(owner, out var ownerEntry)) + ResolveEntry(owner, ownerEntry); + } + } + + break; + + case ReadKind.DirectoryData: + var directory = read.Directory!; + Array.Copy(sectors, 0, directory.Data, read.DirectoryOffset, Math.Min(read.Length, sectors.Length)); + if (--directory.Pending == 0) + CompleteDirectory(directory.Key, directory.Data); + + break; + + default: + // All read kinds are handled above + break; + } + } + + /// + /// Queue what a File Entry still needs: its Allocation Extent Descriptors, and its contents if it is a directory + /// + private void ResolveEntry(long key, FileEntry entry) + { + var descriptors = _volume.GetAllocationDescriptors(key, entry, out long missingKey); + if (descriptors is null) + { + if (missingKey >= 0) + QueueAllocationExtent(missingKey, key); + + return; + } + + if (entry.ICBTag.FileType != FileType.DIRECTORY || _volume.Directories.ContainsKey(key)) + return; + + int length = (int)entry.InformationLength; + if (entry.ICBTag.GetAllocationDescriptorType() == AllocationDescriptorType.EMBEDDED) + { + if (length <= entry.AllocationDescriptors.Length) + CompleteDirectory(key, entry.AllocationDescriptors.Length == length ? entry.AllocationDescriptors : Slice(entry.AllocationDescriptors, 0, length)); + + return; + } + + // Queue the recorded extents; extents that are not recorded read as zeros + var directory = new DirectoryContents { Key = key, Data = new byte[length] }; + List reads = []; + int offset = 0; + foreach (var ad in descriptors) + { + if (offset >= length) + break; + + int extentLength = (int)Math.Min(UDFExtensions.GetExtentLength(ad.ExtentLength), (uint)(length - offset)); + if (UDFExtensions.GetExtentType(ad.ExtentLength) == ExtentType.RECORDED_AND_ALLOCATED) + { + if (!_volume.TryGetSector(ad.ExtentLocation.GetKey(), out long sector)) + return; + + int count = (extentLength + Constants.SectorSize - 1) / Constants.SectorSize; + reads.Add(new PendingRead + { + Sector = sector, + Count = count, + Kind = ReadKind.DirectoryData, + Key = key, + Directory = directory, + DirectoryOffset = offset, + Length = extentLength, + }); + } + + offset += extentLength; + } + + directory.Pending = reads.Count; + if (reads.Count == 0) + { + CompleteDirectory(key, directory.Data); + return; + } + + foreach (var read in reads) + { + Queue(read); + } + } + + /// + /// Queue an Allocation Extent Descriptor needed by a File Entry + /// + private void QueueAllocationExtent(long key, long owner) + { + if (_queuedExtents.TryGetValue(key, out var owners)) + { + if (!owners.Contains(owner)) + owners.Add(owner); + + return; + } + + _queuedExtents[key] = [owner]; + if (_volume.TryGetSector(key, out long sector)) + Queue(new PendingRead { Sector = sector, Count = 1, Kind = ReadKind.AllocationExtent, Key = key }); + } + + /// + /// Parse the contents of a directory and queue the File Entries it identifies + /// + private void CompleteDirectory(long key, byte[] contents) + { + var fids = ParseDirectory(contents, contents.Length); + if (fids is null) + return; + + _volume.Directories[key] = fids; + if (!_includeDirectories) + return; + + foreach (var fid in fids) + { + if ((fid.FileCharacteristics & (FileCharacteristics.DELETED | FileCharacteristics.PARENT)) != 0) + continue; + + QueueEntry(fid.ICB.ExtentLocation.GetKey()); + } + } + + /// + /// Read sectors alone, null if they cannot be read + /// + private byte[]? TryReadSectors(long sector, int count) + { + try + { + return ReadSectors(_data, sector, count); + } + catch (IOException) + { + return null; + } + } + + /// + /// Copy part of an array + /// + private static byte[] Slice(byte[] data, int offset, int length) + { + var slice = new byte[length]; + Array.Copy(data, offset, slice, 0, length); + return slice; + } + } + + #endregion + } +} diff --git a/SabreTools.Wrappers.Test/SabreTools.Wrappers.Test.csproj b/SabreTools.Wrappers.Test/SabreTools.Wrappers.Test.csproj index 3d9b08c3..c2a79871 100644 --- a/SabreTools.Wrappers.Test/SabreTools.Wrappers.Test.csproj +++ b/SabreTools.Wrappers.Test/SabreTools.Wrappers.Test.csproj @@ -32,6 +32,10 @@ + + + + diff --git a/SabreTools.Wrappers.Test/UDFTests.cs b/SabreTools.Wrappers.Test/UDFTests.cs new file mode 100644 index 00000000..492e5b23 --- /dev/null +++ b/SabreTools.Wrappers.Test/UDFTests.cs @@ -0,0 +1,162 @@ +using System.IO; +using System.Linq; +using SabreTools.Serialization.Readers.Test; +using Xunit; + +namespace SabreTools.Wrappers.Test +{ + public class UDFTests + { + [Fact] + public void NullArray_Null() + { + byte[]? data = null; + int offset = 0; + var actual = UDF.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void EmptyArray_Null() + { + byte[]? data = []; + int offset = 0; + var actual = UDF.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void InvalidArray_Null() + { + byte[]? data = [.. Enumerable.Repeat(0xFF, 1024)]; + int offset = 0; + var actual = UDF.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void NullStream_Null() + { + Stream? data = null; + var actual = UDF.Create(data); + Assert.Null(actual); + } + + [Fact] + public void EmptyStream_Null() + { + Stream? data = new MemoryStream([]); + var actual = UDF.Create(data); + Assert.Null(actual); + } + + [Fact] + public void InvalidStream_Null() + { + Stream? data = new MemoryStream([.. Enumerable.Repeat(0xFF, 1024)]); + var actual = UDF.Create(data); + Assert.Null(actual); + } + + [Fact] + public void DirectoryLoop_Extract() + { + var wrapper = UDF.Create(new MemoryStream(UDFImageBuilder.Build(subLoop: true))); + Assert.NotNull(wrapper); + + string output = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + try + { + Directory.CreateDirectory(output); + Assert.True(wrapper.Extract(output, false)); + Assert.Equal(UDFImageBuilder.DataBinLength, new FileInfo(Path.Combine(output, "Sub", "data.bin")).Length); + Assert.True(Directory.Exists(Path.Combine(output, "Sub", "loop"))); + } + finally + { + if (Directory.Exists(output)) + Directory.Delete(output, true); + } + } + + [Fact] + public void ParentDirectoryName_NotExtracted() + { + // "Sub" is named "..", so "data.bin" would be written next to the output directory + var wrapper = UDF.Create(new MemoryStream(UDFImageBuilder.Build(subName: ".."))); + Assert.NotNull(wrapper); + + string parent = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + string output = Path.Combine(parent, "output"); + try + { + Directory.CreateDirectory(output); + Assert.False(wrapper.Extract(output, false)); + Assert.False(File.Exists(Path.Combine(parent, "data.bin"))); + Assert.True(File.Exists(Path.Combine(output, "README.TXT"))); + } + finally + { + if (Directory.Exists(parent)) + Directory.Delete(parent, true); + } + } + + [Fact] + public void RecordedPastEnd_OtherFilesExtracted() + { + // "Media" is recorded in a partition block past the end of the image + var wrapper = UDF.Create(new MemoryStream(UDFImageBuilder.Build(mediaPastEnd: true))); + Assert.NotNull(wrapper); + + string output = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + try + { + Directory.CreateDirectory(output); + Assert.False(wrapper.Extract(output, false)); + Assert.True(File.Exists(Path.Combine(output, "README.TXT"))); + Assert.Equal(UDFImageBuilder.DataBinLength, new FileInfo(Path.Combine(output, "Sub", "data.bin")).Length); + } + finally + { + if (Directory.Exists(output)) + Directory.Delete(output, true); + } + } + + [Fact] + public void StreamAtOffset_VolumeAtOffsetExtracted() + { + // Another volume comes first, so reading from the start of the stream would find "Outer" instead of "Inner" + var stream = new MemoryStream([.. UDFImageBuilder.Build(subName: "Outer"), .. UDFImageBuilder.Build(subName: "Inner")]); + stream.Position = UDFImageBuilder.Sectors * UDFImageBuilder.SectorSize; + + var wrapper = UDF.Create(stream); + Assert.NotNull(wrapper); + + string output = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + try + { + Directory.CreateDirectory(output); + Assert.True(wrapper.Extract(output, false)); + Assert.Equal(UDFImageBuilder.DataBinLength, new FileInfo(Path.Combine(output, "Inner", "data.bin")).Length); + Assert.False(Directory.Exists(Path.Combine(output, "Outer"))); + } + finally + { + if (Directory.Exists(output)) + Directory.Delete(output, true); + } + } + + [Fact] + public void StreamAtOffset_WrapperFactoryUDF() + { + var stream = new MemoryStream([.. new byte[3 * UDFImageBuilder.SectorSize], .. UDFImageBuilder.Build()]); + stream.Position = 3 * UDFImageBuilder.SectorSize; + + var wrapper = WrapperFactory.CreateDiscImageWrapper(stream); + Assert.IsType(wrapper); + } + } +} diff --git a/SabreTools.Wrappers/UDF.Extraction.cs b/SabreTools.Wrappers/UDF.Extraction.cs new file mode 100644 index 00000000..05e4d4f2 --- /dev/null +++ b/SabreTools.Wrappers/UDF.Extraction.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SabreTools.Data.Extensions; +using SabreTools.Data.Models.UDF; +using SabreTools.IO.Extensions; +using SabreTools.Numerics.Extensions; + +namespace SabreTools.Wrappers +{ + public partial class UDF : IExtractable + { + /// + public bool Extract(string outputDirectory, bool includeDebug) + { + // If we have no file set or directories, there is nothing to extract + if (FileSetDescriptor is null || Directories.Count == 0) + return true; + + long root = FileSetDescriptor.RootDirectoryICB.ExtentLocation.GetKey(); + var visited = new HashSet { root }; + return ExtractDirectory(root, Path.GetFullPath(outputDirectory), visited, includeDebug); + } + + /// + /// Extract all files and directories within a directory + /// + private bool ExtractDirectory(long key, string outputPath, HashSet visited, bool includeDebug) + { + if (!Directories.TryGetValue(key, out var fids)) + return false; + + bool succeeded = true; + foreach (var fid in fids) + { + // Skip deleted entries and the parent directory + if ((fid.FileCharacteristics & (FileCharacteristics.DELETED | FileCharacteristics.PARENT)) != 0) + continue; + + string? name = fid.FileIdentifier.DecodeDCharacters(); + if (!IsSafeName(name)) + { + if (includeDebug) Console.WriteLine($"Skipping entry with unsafe name in {outputPath}"); + + succeeded = false; + continue; + } + + long childKey = fid.ICB.ExtentLocation.GetKey(); + if (!FileEntries.TryGetValue(childKey, out var entry)) + { + if (includeDebug) Console.WriteLine($"File Entry missing, cannot extract: {name}"); + + succeeded = false; + continue; + } + + string childPath = Path.Combine(outputPath, name!); + if (entry.ICBTag.FileType == FileType.DIRECTORY) + { + if (includeDebug) Console.WriteLine($"Extracting to directory: {childPath}"); + if (!Directory.Exists(childPath)) + Directory.CreateDirectory(childPath); + + // Directories reachable twice (hard links, loops) are extracted once + if (visited.Add(childKey)) + succeeded &= ExtractDirectory(childKey, childPath, visited, includeDebug); + } + else + { + succeeded &= ExtractFile(childKey, entry, childPath, includeDebug); + } + } + + return succeeded; + } + + /// + /// Extract a file's data + /// + private bool ExtractFile(long key, FileEntry entry, string filePath, bool includeDebug) + { + if (File.Exists(filePath) || Directory.Exists(filePath)) + { + if (includeDebug) Console.WriteLine($"File/Folder already exists, cannot extract: {filePath}"); + return false; + } + + if (includeDebug) Console.WriteLine($"Extracting: {filePath}"); + + long size = (long)entry.InformationLength; + if (entry.ICBTag.GetAllocationDescriptorType() == AllocationDescriptorType.EMBEDDED) + { + using var embedded = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None); + embedded.Write(entry.AllocationDescriptors, 0, (int)size); + return true; + } + + var descriptors = Model.GetAllocationDescriptors(key, entry); + if (descriptors is null) + { + if (includeDebug) Console.WriteLine($"Allocation descriptors not supported or incomplete: {filePath}"); + return false; + } + + const int chunkSize = 2048 * 1024; + using var fs = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None); + long written = 0; + foreach (var ad in descriptors) + { + if (written >= size) + break; + + long length = Math.Min(UDFExtensions.GetExtentLength(ad.ExtentLength), size - written); + bool recorded = UDFExtensions.GetExtentType(ad.ExtentLength) == ExtentType.RECORDED_AND_ALLOCATED; + + long sector = 0; + if (recorded && !Model.TryGetSector(ad.ExtentLocation.GetKey(), out sector)) + return false; + + // Make sure the extent won't EOF + if (recorded && (sector * Constants.SectorSize) + length > _dataSource.Length) + return false; + + long done = 0; + while (done < length) + { + int count = (int)Math.Min(chunkSize, length - done); + byte[] buffer; + if (recorded) + { + lock (_dataSourceLock) + { + _dataSource.SeekIfPossible((sector * Constants.SectorSize) + done, SeekOrigin.Begin); + buffer = _dataSource.ReadBytes(count); + } + } + else + { + // Extents that are not recorded read as zeros + buffer = new byte[count]; + } + + fs.Write(buffer, 0, count); + done += count; + } + + written += length; + } + + return written == size; + } + + /// + /// Check that a file identifier can be used as a single path component + /// + private static bool IsSafeName(string? name) + { + if (string.IsNullOrEmpty(name) || name == "." || name == "..") + return false; + + char[] invalid = Path.GetInvalidFileNameChars(); + foreach (char ch in name!) + { + if (ch == '/' || ch == '\\' || ch == '\0' || Array.Exists(invalid, x => x == ch)) + return false; + } + + return true; + } + } +} diff --git a/SabreTools.Wrappers/UDF.Printing.cs b/SabreTools.Wrappers/UDF.Printing.cs new file mode 100644 index 00000000..c7168da6 --- /dev/null +++ b/SabreTools.Wrappers/UDF.Printing.cs @@ -0,0 +1,296 @@ +using System.Collections.Generic; +using System.Text; +using SabreTools.Data.Extensions; +using SabreTools.Data.Models.UDF; +using SabreTools.Text.Extensions; + +namespace SabreTools.Wrappers +{ + public partial class UDF : IPrintable + { +#if NETCOREAPP + /// + public string ExportJSON(bool recursive) => System.Text.Json.JsonSerializer.Serialize(Model, _jsonSerializerOptions); +#else + /// + public string ExportJSON(bool recursive) => Newtonsoft.Json.JsonConvert.SerializeObject(Model, _jsonSerializerOptions); +#endif + + /// + public void PrintInformation(StringBuilder builder, bool recursive) + { + builder.AppendLine("UDF Information:"); + builder.AppendLine("-------------------------"); + builder.AppendLine(); + + Print(builder, Model.VolumeRecognitionSequence); + Print(builder, Model.AnchorVolumeDescriptorPointer); + Print(builder, Model.VolumeDescriptorSequence); + Print(builder, Model.FileSetDescriptor); + + if (Model.FileSetDescriptor is not null) + { + builder.AppendLine(" Directory Hierarchy:"); + builder.AppendLine(" -------------------------"); + long root = Model.FileSetDescriptor.RootDirectoryICB.ExtentLocation.GetKey(); + PrintDirectory(builder, root, "\\", new HashSet { root }); + builder.AppendLine(); + } + } + + private static void Print(StringBuilder builder, VolumeStructureDescriptor[] sequence) + { + builder.AppendLine(" Volume Recognition Sequence:"); + builder.AppendLine(" -------------------------"); + if (sequence.Length == 0) + { + builder.AppendLine(" No volume structure descriptors"); + builder.AppendLine(); + return; + } + + for (int i = 0; i < sequence.Length; i++) + { + var vsd = sequence[i]; + builder.AppendLine($" Volume Structure Descriptor {i}:"); + builder.AppendLine(vsd.StructureType, " Structure Type"); + builder.AppendLine(Encoding.ASCII.GetString(vsd.StandardIdentifier), " Standard Identifier"); + builder.AppendLine(vsd.StructureVersion, " Structure Version"); + } + + builder.AppendLine(); + } + + private static void Print(StringBuilder builder, AnchorVolumeDescriptorPointer? anchor) + { + builder.AppendLine(" Anchor Volume Descriptor Pointer:"); + builder.AppendLine(" -------------------------"); + if (anchor is null) + { + builder.AppendLine(" No anchor volume descriptor pointer"); + builder.AppendLine(); + return; + } + + Print(builder, anchor.DescriptorTag, " "); + Print(builder, anchor.MainVolumeDescriptorSequenceExtent, " Main Volume Descriptor Sequence"); + Print(builder, anchor.ReserveVolumeDescriptorSequenceExtent, " Reserve Volume Descriptor Sequence"); + builder.AppendLine(); + } + + private static void Print(StringBuilder builder, Descriptor[] sequence) + { + builder.AppendLine(" Volume Descriptor Sequence:"); + builder.AppendLine(" -------------------------"); + if (sequence.Length == 0) + { + builder.AppendLine(" No volume descriptors"); + builder.AppendLine(); + return; + } + + foreach (var descriptor in sequence) + { + builder.AppendLine($" {descriptor.DescriptorTag.TagIdentifier} (Sector {descriptor.DescriptorTag.TagLocation}):"); + Print(builder, descriptor.DescriptorTag, " "); + switch (descriptor) + { + case PrimaryVolumeDescriptor pvd: + builder.AppendLine(pvd.VolumeDescriptorSequenceNumber, " Volume Descriptor Sequence Number"); + builder.AppendLine(pvd.PrimaryVolumeDescriptorNumber, " Primary Volume Descriptor Number"); + builder.AppendLine(pvd.VolumeIdentifier.DecodeDString(), " Volume Identifier"); + builder.AppendLine(pvd.VolumeSequenceNumber, " Volume Sequence Number"); + builder.AppendLine(pvd.MaximumVolumeSequenceNumber, " Maximum Volume Sequence Number"); + builder.AppendLine(pvd.InterchangeLevel, " Interchange Level"); + builder.AppendLine(pvd.MaximumInterchangeLevel, " Maximum Interchange Level"); + builder.AppendLine(pvd.VolumeSetIdentifier.DecodeDString(), " Volume Set Identifier"); + Print(builder, pvd.ApplicationIdentifier, " Application Identifier"); + builder.AppendLine(Format(pvd.RecordingDateAndTime), " Recording Date and Time"); + Print(builder, pvd.ImplementationIdentifier, " Implementation Identifier"); + builder.AppendLine(pvd.Flags, " Flags"); + break; + + case PartitionDescriptor pd: + builder.AppendLine(pd.VolumeDescriptorSequenceNumber, " Volume Descriptor Sequence Number"); + builder.AppendLine(pd.PartitionFlags, " Partition Flags"); + builder.AppendLine(pd.PartitionNumber, " Partition Number"); + Print(builder, pd.PartitionContents, " Partition Contents"); + builder.AppendLine(pd.AccessType, " Access Type"); + builder.AppendLine(pd.PartitionStartingLocation, " Partition Starting Location"); + builder.AppendLine(pd.PartitionLength, " Partition Length"); + Print(builder, pd.ImplementationIdentifier, " Implementation Identifier"); + break; + + case LogicalVolumeDescriptor lvd: + builder.AppendLine(lvd.VolumeDescriptorSequenceNumber, " Volume Descriptor Sequence Number"); + builder.AppendLine(lvd.LogicalVolumeIdentifier.DecodeDString(), " Logical Volume Identifier"); + builder.AppendLine(lvd.LogicalBlockSize, " Logical Block Size"); + Print(builder, lvd.DomainIdentifier, " Domain Identifier"); + Print(builder, lvd.LogicalVolumeContentsUse, " File Set Descriptor Extent"); + builder.AppendLine(lvd.MapTableLength, " Map Table Length"); + builder.AppendLine(lvd.NumberOfPartitionMaps, " Number of Partition Maps"); + Print(builder, lvd.ImplementationIdentifier, " Implementation Identifier"); + Print(builder, lvd.IntegritySequenceExtent, " Integrity Sequence Extent"); + for (int i = 0; i < lvd.PartitionMaps.Length; i++) + { + Print(builder, lvd.PartitionMaps[i], i); + } + + break; + + case VolumeDescriptorPointer vdp: + builder.AppendLine(vdp.VolumeDescriptorSequenceNumber, " Volume Descriptor Sequence Number"); + Print(builder, vdp.NextVolumeDescriptorSequenceExtent, " Next Volume Descriptor Sequence"); + break; + + default: + // Other descriptors are not parsed further + break; + } + } + + builder.AppendLine(); + } + + private static void Print(StringBuilder builder, PartitionMap map, int index) + { + builder.AppendLine($" Partition Map {index} (Partition Reference Number {index}):"); + builder.AppendLine(map.PartitionMapType, " Partition Map Type"); + builder.AppendLine(map.PartitionMapLength, " Partition Map Length"); + switch (map) + { + case Type1PartitionMap type1: + builder.AppendLine(type1.VolumeSequenceNumber, " Volume Sequence Number"); + builder.AppendLine(type1.PartitionNumber, " Partition Number"); + break; + + case MetadataPartitionMap metadata: + Print(builder, metadata.PartitionTypeIdentifier, " Partition Type Identifier"); + builder.AppendLine(metadata.VolumeSequenceNumber, " Volume Sequence Number"); + builder.AppendLine(metadata.PartitionNumber, " Partition Number"); + builder.AppendLine(metadata.MetadataFileLocation, " Metadata File Location"); + builder.AppendLine(metadata.MetadataMirrorFileLocation, " Metadata Mirror File Location"); + builder.AppendLine(metadata.MetadataBitmapFileLocation, " Metadata Bitmap File Location"); + builder.AppendLine(metadata.AllocationUnitSize, " Allocation Unit Size"); + builder.AppendLine(metadata.AlignmentUnitSize, " Alignment Unit Size"); + builder.AppendLine(metadata.Flags, " Flags"); + break; + + case Type2PartitionMap type2: + Print(builder, type2.PartitionTypeIdentifier, " Partition Type Identifier"); + builder.AppendLine(type2.VolumeSequenceNumber, " Volume Sequence Number"); + builder.AppendLine(type2.PartitionNumber, " Partition Number"); + builder.AppendLine(type2.PartitionTypeData, " Partition Type Data"); + break; + + default: + // Unknown partition map types are not parsed further + break; + } + } + + private static void Print(StringBuilder builder, FileSetDescriptor? fsd) + { + builder.AppendLine(" File Set Descriptor:"); + builder.AppendLine(" -------------------------"); + if (fsd is null) + { + builder.AppendLine(" No file set descriptor"); + builder.AppendLine(); + return; + } + + Print(builder, fsd.DescriptorTag, " "); + builder.AppendLine(Format(fsd.RecordingDateAndTime), " Recording Date and Time"); + builder.AppendLine(fsd.InterchangeLevel, " Interchange Level"); + builder.AppendLine(fsd.MaximumInterchangeLevel, " Maximum Interchange Level"); + builder.AppendLine(fsd.FileSetNumber, " File Set Number"); + builder.AppendLine(fsd.FileSetDescriptorNumber, " File Set Descriptor Number"); + builder.AppendLine(fsd.LogicalVolumeIdentifier.DecodeDString(), " Logical Volume Identifier"); + builder.AppendLine(fsd.FileSetIdentifier.DecodeDString(), " File Set Identifier"); + builder.AppendLine(fsd.CopyrightFileIdentifier.DecodeDString(), " Copyright File Identifier"); + builder.AppendLine(fsd.AbstractFileIdentifier.DecodeDString(), " Abstract File Identifier"); + Print(builder, fsd.RootDirectoryICB, " Root Directory ICB"); + Print(builder, fsd.DomainIdentifier, " Domain Identifier"); + Print(builder, fsd.NextExtent, " Next Extent"); + Print(builder, fsd.SystemStreamDirectoryICB, " System Stream Directory ICB"); + builder.AppendLine(); + } + + private void PrintDirectory(StringBuilder builder, long key, string path, HashSet visited) + { + if (!Model.Directories.TryGetValue(key, out var fids)) + { + builder.AppendLine($" {path} (directory not read)"); + return; + } + + foreach (var fid in fids) + { + if ((fid.FileCharacteristics & (FileCharacteristics.DELETED | FileCharacteristics.PARENT)) != 0) + continue; + + string name = fid.FileIdentifier.DecodeDCharacters() ?? "(unknown encoding)"; + string childPath = path + name; + long childKey = fid.ICB.ExtentLocation.GetKey(); + string hidden = (fid.FileCharacteristics & FileCharacteristics.EXISTENCE) != 0 ? ", hidden" : string.Empty; + if (!Model.FileEntries.TryGetValue(childKey, out var entry)) + { + builder.AppendLine($" {childPath} (File Entry not read{hidden})"); + continue; + } + + string location = $"partition {fid.ICB.ExtentLocation.PartitionReferenceNumber}, block {fid.ICB.ExtentLocation.LogicalBlockNumber}"; + if (entry.ICBTag.FileType == FileType.DIRECTORY) + { + builder.AppendLine($" {childPath}\\ ({location}{hidden})"); + if (visited.Add(childKey)) + PrintDirectory(builder, childKey, $"{childPath}\\", visited); + } + else + { + builder.AppendLine($" {childPath} ({entry.InformationLength} bytes, {entry.ICBTag.FileType}, {location}{hidden})"); + } + } + } + + private static void Print(StringBuilder builder, DescriptorTag tag, string indent) + { + builder.AppendLine((ushort)tag.TagIdentifier, $"{indent}Tag Identifier"); + builder.AppendLine(tag.DescriptorVersion, $"{indent}Descriptor Version"); + builder.AppendLine(tag.TagSerialNumber, $"{indent}Tag Serial Number"); + builder.AppendLine(tag.DescriptorCRCLength, $"{indent}Descriptor CRC Length"); + builder.AppendLine(tag.TagLocation, $"{indent}Tag Location"); + } + + private static void Print(StringBuilder builder, ExtentDescriptor extent, string label) + => builder.AppendLine($"sector {extent.ExtentLocation}, {extent.ExtentLength} bytes", label); + + private static void Print(StringBuilder builder, LongAllocationDescriptor ad, string label) + { + uint length = UDFExtensions.GetExtentLength(ad.ExtentLength); + builder.AppendLine($"partition {ad.ExtentLocation.PartitionReferenceNumber}, block {ad.ExtentLocation.LogicalBlockNumber}, {length} bytes", label); + } + + private static void Print(StringBuilder builder, EntityIdentifier identifier, string label) + { + builder.AppendLine(identifier.Identifier, label, Encoding.ASCII); + builder.AppendLine(identifier.Flags, $"{label} Flags"); + builder.AppendLine(identifier.IdentifierSuffix, $"{label} Suffix"); + } + + private static string Format(Timestamp timestamp) + { + if (timestamp.Year == 0 && timestamp.Month == 0 && timestamp.Day == 0) + return "Not specified"; + + // Lower 12 bits: signed offset from UTC in minutes, -2047 = not specified + int offset = timestamp.TypeAndTimezone & 0x0FFF; + if ((offset & 0x0800) != 0) + offset -= 0x1000; + + string zone = offset == -2047 ? string.Empty : $" UTC{(offset < 0 ? "-" : "+")}{System.Math.Abs(offset) / 60:D2}:{System.Math.Abs(offset) % 60:D2}"; + return $"{timestamp.Year:D4}-{timestamp.Month:D2}-{timestamp.Day:D2} {timestamp.Hour:D2}:{timestamp.Minute:D2}:{timestamp.Second:D2}.{timestamp.Centiseconds:D2}{zone}"; + } + } +} diff --git a/SabreTools.Wrappers/UDF.cs b/SabreTools.Wrappers/UDF.cs new file mode 100644 index 00000000..d7ec69f0 --- /dev/null +++ b/SabreTools.Wrappers/UDF.cs @@ -0,0 +1,112 @@ +using System.Collections.Generic; +using System.IO; +using SabreTools.Data.Models.UDF; +using SabreTools.IO; + +namespace SabreTools.Wrappers +{ + public partial class UDF : WrapperBase + { + #region Descriptive Properties + + /// + public override string DescriptionString => "Universal Disk Format (UDF) Volume"; + + #endregion + + #region Extension Properties + + /// + public VolumeStructureDescriptor[] VolumeRecognitionSequence => Model.VolumeRecognitionSequence; + + /// + public Descriptor[] VolumeDescriptorSequence => Model.VolumeDescriptorSequence; + + /// + public FileSetDescriptor? FileSetDescriptor => Model.FileSetDescriptor; + + /// + public Dictionary FileEntries => Model.FileEntries; + + /// + public Dictionary Directories => Model.Directories; + + #endregion + + #region Constructors + + /// + public UDF(Volume model, byte[] data) : base(model, data) { } + + /// + public UDF(Volume model, byte[] data, int offset) : base(model, data, offset) { } + + /// + public UDF(Volume model, byte[] data, int offset, int length) : base(model, data, offset, length) { } + + /// + public UDF(Volume model, Stream data) : base(model, data) { } + + /// + public UDF(Volume model, Stream data, long offset) : base(model, data, offset) { } + + /// + public UDF(Volume model, Stream data, long offset, long length) : base(model, data, offset, length) { } + + #endregion + + #region Static Constructors + + /// + /// Create a UDF Volume from a byte array and offset + /// + /// Byte array representing the UDF Volume + /// Offset within the array to parse + /// A UDF Volume wrapper on success, null on failure + public static UDF? Create(byte[]? data, int offset) + { + // If the data is invalid + if (data is null || data.Length == 0) + return null; + + // If the offset is out of bounds + if (offset < 0 || offset >= data.Length) + return null; + + // Create a memory stream and use that + var dataStream = new MemoryStream(data, offset, data.Length - offset); + return Create(dataStream); + } + + /// + /// Create a UDF Volume from a Stream + /// + /// Stream representing the UDF Volume + /// A UDF Volume wrapper on success, null on failure + public static UDF? Create(Stream? data) + { + // If the data is invalid + if (data is null || !data.CanRead || !data.CanSeek) + return null; + + try + { + // Cache the current offset + long currentOffset = data.Position; + + // The reader addresses logical sectors from the start of the stream, so it reads a view that starts here + var model = new Serialization.Readers.UDF().Deserialize(new ReadOnlyViewStream(data, currentOffset)); + if (model is null) + return null; + + return new UDF(model, data, currentOffset); + } + catch + { + return null; + } + } + + #endregion + } +} diff --git a/SabreTools.Wrappers/WrapperFactory.cs b/SabreTools.Wrappers/WrapperFactory.cs index 44cc24ec..6c72fd26 100644 --- a/SabreTools.Wrappers/WrapperFactory.cs +++ b/SabreTools.Wrappers/WrapperFactory.cs @@ -161,6 +161,14 @@ namespace SabreTools.Wrappers // Reset position in stream stream.SeekIfPossible(initialOffset, SeekOrigin.Begin); + // Try to get a UDF wrapper (discs without an ISO9660 file system) + var udfWrapper = UDF.Create(stream); + if (udfWrapper is not null) + return udfWrapper; + + // Reset position in stream + stream.SeekIfPossible(initialOffset, SeekOrigin.Begin); + // Try to get a 3DO / M2 disc image wrapper var operaFSWrapper = OperaFS.Create(stream); if (operaFSWrapper is not null)