Use new index when saving aaruformat images with more than 65535 blocks. Fixes #326.

Aaru.Images/AaruFormat/Constants.cs

@@ -42,7 +42,9 @@ namespace Aaru.DiscImages
         ///     Image format version. A change in this number indicates an incompatible change to the format that prevents
         ///     older implementations from reading it correctly, if at all.
         /// </summary>
-        const byte AARUFMT_VERSION = 1;
+        const byte AARUFMT_VERSION_V1 = 1;
+        /// <summary>Adds new index format with 64-bit entries counter</summary>
+        const byte AARUFMT_VERSION = 2;
         /// <summary>Maximum read cache size, 256MiB.</summary>
         const uint MAX_CACHE_SIZE = 256 * 1024 * 1024;
         /// <summary>Size in bytes of LZMA properties.</summary>

Aaru.Images/AaruFormat/Enums.cs

@@ -218,9 +218,11 @@ namespace Aaru.DiscImages
             /// <summary>Block containing data</summary>
             DataBlock = 0x4B4C4244,
             /// <summary>Block containing a deduplication table</summary>
-            DeDuplicationTable = 0X2A544444,
+            DeDuplicationTable = 0x2A544444,
             /// <summary>Block containing the index</summary>
-            Index = 0X58444E49,
+            Index = 0x58444E49,
+            /// <summary>Block containing the index</summary>
+            Index2 = 0x32584449,
             /// <summary>Block containing logical geometry</summary>
             GeometryBlock = 0x4D4F4547,
             /// <summary>Block containing metadata</summary>

Aaru.Images/AaruFormat/Read.cs

@@ -86,28 +86,62 @@ namespace Aaru.DiscImages
             imageStream.Read(structureBytes, 0, structureBytes.Length);
             IndexHeader idxHeader = Marshal.SpanToStructureLittleEndian<IndexHeader>(structureBytes);
 
-            if(idxHeader.identifier != BlockType.Index)
+            if(idxHeader.identifier != BlockType.Index &&
+               idxHeader.identifier != BlockType.Index2)
                 throw new FeatureUnsupportedImageException("Index not found!");
 
-            AaruConsole.DebugWriteLine("Aaru Format plugin", "Index at {0} contains {1} entries", header.indexOffset,
+            if(idxHeader.identifier == BlockType.Index2)
-                                       idxHeader.entries);
-
-            AaruConsole.DebugWriteLine("Aaru Format plugin", "Memory snapshot: {0} bytes", GC.GetTotalMemory(false));
-
-            // Fill in-memory index
-            index = new List<IndexEntry>();
-
-            for(ushort i = 0; i < idxHeader.entries; i++)
             {
-                structureBytes = new byte[Marshal.SizeOf<IndexEntry>()];
+                imageStream.Position = (long)header.indexOffset;
+                structureBytes       = new byte[Marshal.SizeOf<IndexHeader2>()];
                 imageStream.Read(structureBytes, 0, structureBytes.Length);
-                IndexEntry entry = Marshal.SpanToStructureLittleEndian<IndexEntry>(structureBytes);
+                IndexHeader2 idxHeader2 = Marshal.SpanToStructureLittleEndian<IndexHeader2>(structureBytes);
 
-                AaruConsole.DebugWriteLine("Aaru Format plugin",
+                AaruConsole.DebugWriteLine("Aaru Format plugin", "Index at {0} contains {1} entries",
-                                           "Block type {0} with data type {1} is indexed to be at {2}", entry.blockType,
+                                           header.indexOffset, idxHeader2.entries);
-                                           entry.dataType, entry.offset);
 
-                index.Add(entry);
+                AaruConsole.DebugWriteLine("Aaru Format plugin", "Memory snapshot: {0} bytes",
+                                           GC.GetTotalMemory(false));
+
+                // Fill in-memory index
+                index = new List<IndexEntry>();
+
+                for(ulong i = 0; i < idxHeader2.entries; i++)
+                {
+                    structureBytes = new byte[Marshal.SizeOf<IndexEntry>()];
+                    imageStream.Read(structureBytes, 0, structureBytes.Length);
+                    IndexEntry entry = Marshal.SpanToStructureLittleEndian<IndexEntry>(structureBytes);
+
+                    AaruConsole.DebugWriteLine("Aaru Format plugin",
+                                               "Block type {0} with data type {1} is indexed to be at {2}",
+                                               entry.blockType, entry.dataType, entry.offset);
+
+                    index.Add(entry);
+                }
+            }
+            else
+            {
+                AaruConsole.DebugWriteLine("Aaru Format plugin", "Index at {0} contains {1} entries",
+                                           header.indexOffset, idxHeader.entries);
+
+                AaruConsole.DebugWriteLine("Aaru Format plugin", "Memory snapshot: {0} bytes",
+                                           GC.GetTotalMemory(false));
+
+                // Fill in-memory index
+                index = new List<IndexEntry>();
+
+                for(ushort i = 0; i < idxHeader.entries; i++)
+                {
+                    structureBytes = new byte[Marshal.SizeOf<IndexEntry>()];
+                    imageStream.Read(structureBytes, 0, structureBytes.Length);
+                    IndexEntry entry = Marshal.SpanToStructureLittleEndian<IndexEntry>(structureBytes);
+
+                    AaruConsole.DebugWriteLine("Aaru Format plugin",
+                                               "Block type {0} with data type {1} is indexed to be at {2}",
+                                               entry.blockType, entry.dataType, entry.offset);
+
+                    index.Add(entry);
+                }
             }
 
             AaruConsole.DebugWriteLine("Aaru Format plugin", "Memory snapshot: {0} bytes", GC.GetTotalMemory(false));

Aaru.Images/AaruFormat/Structs.cs

@@ -101,6 +101,18 @@ namespace Aaru.DiscImages
             public ulong crc64;
         }
 
+        /// <summary>Header for the index, followed by entries</summary>
+        [StructLayout(LayoutKind.Sequential, Pack = 1)]
+        struct IndexHeader2
+        {
+            /// <summary>Identifier, <see cref="BlockType.Index2" /></summary>
+            public BlockType identifier;
+            /// <summary>How many entries follow this header</summary>
+            public ulong entries;
+            /// <summary>CRC64-ECMA of the index</summary>
+            public ulong crc64;
+        }
+
         /// <summary>Index entry</summary>
         [StructLayout(LayoutKind.Sequential, Pack = 1)]
         struct IndexEntry

Aaru.Images/AaruFormat/Write.cs

@@ -270,7 +270,7 @@ namespace Aaru.DiscImages
             }
 
             header.application             = "Aaru";
-            header.imageMajorVersion       = AARUFMT_VERSION;
+            header.imageMajorVersion       = AARUFMT_VERSION_V1;
             header.imageMinorVersion       = 0;
             header.applicationMajorVersion = (byte)typeof(AaruFormat).Assembly.GetName().Version.Major;
             header.applicationMinorVersion = (byte)typeof(AaruFormat).Assembly.GetName().Version.Minor;
@@ -3357,15 +3357,36 @@ namespace Aaru.DiscImages
 
             Crc64Context.Data(blockStream.ToArray(), out byte[] idxCrc);
 
-            var idxHeader = new IndexHeader
+            if(index.Count > ushort.MaxValue)
             {
-                identifier = BlockType.Index, entries = (ushort)index.Count, crc64 = BitConverter.ToUInt64(idxCrc, 0)
+                header.imageMajorVersion = AARUFMT_VERSION;
-            };
+
+                var idxHeader = new IndexHeader2
+                {
+                    identifier = BlockType.Index2, entries = (ulong)index.Count,
+                    crc64      = BitConverter.ToUInt64(idxCrc, 0)
+                };
+
+                // Write index header to disk
+                structureBytes = new byte[Marshal.SizeOf<IndexHeader2>()];
+                MemoryMarshal.Write(structureBytes, ref idxHeader);
+                imageStream.Write(structureBytes, 0, structureBytes.Length);
+            }
+            else
+            {
+                var idxHeader = new IndexHeader
+                {
+                    identifier = BlockType.Index, entries = (ushort)index.Count,
+                    crc64      = BitConverter.ToUInt64(idxCrc, 0)
+                };
+
+                // Write index header to disk
+                structureBytes = new byte[Marshal.SizeOf<IndexHeader>()];
+                MemoryMarshal.Write(structureBytes, ref idxHeader);
+                imageStream.Write(structureBytes, 0, structureBytes.Length);
+            }
 
             // Write index to disk
-            structureBytes = new byte[Marshal.SizeOf<IndexHeader>()];
-            MemoryMarshal.Write(structureBytes, ref idxHeader);
-            imageStream.Write(structureBytes, 0, structureBytes.Length);
             imageStream.Write(blockStream.ToArray(), 0, (int)blockStream.Length);
             blockStream.Close();
             blockStream = null;