For DiscImageChef format, add image and sector verification.

DiscImageChef.DiscImages/DiscImageChef.cs

@@ -296,7 +296,6 @@ namespace DiscImageChef.DiscImages
                             lzmaBlock.Read(data, 0, (int)blockHeader.length);
                             lzmaBlock.Close();
                             compressedTagMs.Close();
-                            compressedTag = null;
                         }
                         else if(blockHeader.compression == CompressionType.None)
                         {
@@ -387,7 +386,6 @@ namespace DiscImageChef.DiscImages
 
                         if(ddtHeader.identifier != BlockType.DeDuplicationTable) break;
 
-                        // TODO: Check CRC64
                         imageInfo.Sectors = ddtHeader.entries;
                         shift             = ddtHeader.shift;
 
@@ -406,11 +404,9 @@ namespace DiscImageChef.DiscImages
                                 lzmaDdt.Read(decompressedDdt, 0, (int)ddtHeader.length);
                                 lzmaDdt.Close();
                                 compressedDdtMs.Close();
-                                compressedDdt = null;
                                 userDataDdt   = new ulong[ddtHeader.entries];
                                 for(ulong i = 0; i < ddtHeader.entries; i++)
                                     userDataDdt[i] = BitConverter.ToUInt64(decompressedDdt, (int)(i * sizeof(ulong)));
-                                decompressedDdt    = null;
                                 DateTime ddtEnd    = DateTime.UtcNow;
                                 inMemoryDdt        = true;
                                 DicConsole.DebugWriteLine("DiscImageChef format plugin",
@@ -634,7 +630,18 @@ namespace DiscImageChef.DiscImages
                             break;
                         }
 
-                        // TODO: Check CRC64
+                        structureBytes = new byte[Marshal.SizeOf(typeof(TrackEntry)) * tracksHeader.entries];
+                        imageStream.Read(structureBytes, 0, structureBytes.Length);
+                        Crc64Context.Data(structureBytes, out byte[] trksCrc);
+                        if(BitConverter.ToUInt64(trksCrc, 0) != tracksHeader.crc64)
+                        {
+                            DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                      "Incorrect CRC found: 0x{0:X16} found, expected 0x{1:X16}, continuing...",
+                                                      BitConverter.ToUInt64(trksCrc, 0), tracksHeader.crc64);
+                            break;
+                        }
+
+                        imageStream.Position -= structureBytes.Length;
 
                         Tracks     = new List<Track>();
                         trackFlags = new Dictionary<byte, byte>();
@@ -838,7 +845,6 @@ namespace DiscImageChef.DiscImages
                     lzmaBlock.Read(block, 0, (int)blockHeader.length);
                     lzmaBlock.Close();
                     compressedBlockMs.Close();
-                    compressedBlock = null;
                     break;
                 default:
                     throw new
@@ -1322,47 +1328,276 @@ namespace DiscImageChef.DiscImages
 
         public bool? VerifySector(ulong sectorAddress)
         {
-            throw new NotImplementedException();
+            if(imageInfo.XmlMediaType != XmlMediaType.OpticalDisc) return null;
+
+            byte[] buffer = ReadSectorLong(sectorAddress);
+            return CdChecksums.CheckCdSector(buffer);
         }
 
         public bool? VerifySector(ulong sectorAddress, uint track)
         {
-            if(imageInfo.XmlMediaType != XmlMediaType.OpticalDisc)
+            if(imageInfo.XmlMediaType != XmlMediaType.OpticalDisc) return null;
-                throw new FeatureNotPresentImageException("Feature not present in image");
 
-            Track trk = Tracks.FirstOrDefault(t => t.TrackSequence == track);
+            byte[] buffer = ReadSectorLong(sectorAddress, track);
-            if(trk.TrackSequence                                   != track)
+            return CdChecksums.CheckCdSector(buffer);
-                throw new ArgumentOutOfRangeException(nameof(track), "Track does not exist in disc image");
-
-            return VerifySector(trk.TrackStartSector + sectorAddress);
         }
 
         public bool? VerifySectors(ulong sectorAddress, uint length, out List<ulong> failingLbas,
                                    out                                   List<ulong> unknownLbas)
         {
-            throw new NotImplementedException();
+            failingLbas = new List<ulong>();
+            unknownLbas = new List<ulong>();
+
+            if(imageInfo.XmlMediaType != XmlMediaType.OpticalDisc)
+            {
+                for(ulong i = sectorAddress; i < sectorAddress + length; i++) unknownLbas.Add(i);
+
+                return null;
+            }
+
+            byte[] buffer = ReadSectorsLong(sectorAddress, length);
+            int    bps    = (int)(buffer.Length / length);
+            byte[] sector = new byte[bps];
+            failingLbas   = new List<ulong>();
+            unknownLbas   = new List<ulong>();
+
+            for(int i = 0; i < length; i++)
+            {
+                Array.Copy(buffer, i * bps, sector, 0, bps);
+                bool? sectorStatus = CdChecksums.CheckCdSector(sector);
+
+                switch(sectorStatus)
+                {
+                    case null:
+                        unknownLbas.Add((ulong)i + sectorAddress);
+                        break;
+                    case false:
+                        failingLbas.Add((ulong)i + sectorAddress);
+                        break;
+                }
+            }
+
+            if(unknownLbas.Count > 0) return null;
+
+            return failingLbas.Count <= 0;
         }
 
         public bool? VerifySectors(ulong sectorAddress, uint length, uint track, out List<ulong> failingLbas,
                                    out                                               List<ulong> unknownLbas)
         {
             if(imageInfo.XmlMediaType != XmlMediaType.OpticalDisc)
-                throw new FeatureNotPresentImageException("Feature not present in image");
+            {
+                failingLbas = new List<ulong>();
+                unknownLbas = new List<ulong>();
 
-            Track trk = Tracks.FirstOrDefault(t => t.TrackSequence == track);
+                for(ulong i = sectorAddress; i < sectorAddress + length; i++) unknownLbas.Add(i);
-            if(trk.TrackSequence                                   != track)
-                throw new ArgumentOutOfRangeException(nameof(track), "Track does not exist in disc image");
 
-            if(trk.TrackStartSector + sectorAddress + length > trk.TrackEndSector + 1)
+                return null;
-                throw new ArgumentOutOfRangeException(nameof(length),
+            }
-                                                      $"Requested more sectors ({length + sectorAddress}) than present in track ({trk.TrackEndSector - trk.TrackStartSector + 1}), won't cross tracks");
 
-            return VerifySectors(trk.TrackStartSector + sectorAddress, length, out failingLbas, out unknownLbas);
+            byte[] buffer = ReadSectorsLong(sectorAddress, length, track);
+            int    bps    = (int)(buffer.Length / length);
+            byte[] sector = new byte[bps];
+            failingLbas   = new List<ulong>();
+            unknownLbas   = new List<ulong>();
+
+            for(int i = 0; i < length; i++)
+            {
+                Array.Copy(buffer, i * bps, sector, 0, bps);
+                bool? sectorStatus = CdChecksums.CheckCdSector(sector);
+
+                switch(sectorStatus)
+                {
+                    case null:
+                        unknownLbas.Add((ulong)i + sectorAddress);
+                        break;
+                    case false:
+                        failingLbas.Add((ulong)i + sectorAddress);
+                        break;
+                }
+            }
+
+            if(unknownLbas.Count > 0) return null;
+
+            return failingLbas.Count <= 0;
         }
 
         public bool? VerifyMediaImage()
         {
-            throw new NotImplementedException();
+            DicConsole.DebugWriteLine("DiscImageChef format plugin", "Checking index integrity at {0}",
+                                      header.indexOffset);
+            imageStream.Position = (long)header.indexOffset;
+
+            IndexHeader idxHeader = new IndexHeader();
+            structureBytes        = new byte[Marshal.SizeOf(idxHeader)];
+            imageStream.Read(structureBytes, 0, structureBytes.Length);
+            structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(idxHeader));
+            Marshal.Copy(structureBytes, 0, structurePointer, Marshal.SizeOf(idxHeader));
+            idxHeader = (IndexHeader)Marshal.PtrToStructure(structurePointer, typeof(IndexHeader));
+            Marshal.FreeHGlobal(structurePointer);
+
+            if(idxHeader.identifier != BlockType.Index)
+            {
+                DicConsole.DebugWriteLine("DiscImageChef format plugin", "Incorrect index identifier");
+                return false;
+            }
+
+            DicConsole.DebugWriteLine("DiscImageChef format plugin", "Index at {0} contains {1} entries",
+                                      header.indexOffset, idxHeader.entries);
+
+            structureBytes = new byte[Marshal.SizeOf(typeof(IndexEntry)) * idxHeader.entries];
+            imageStream.Read(structureBytes, 0, structureBytes.Length);
+            Crc64Context.Data(structureBytes, out byte[] verifyCrc);
+
+            if(BitConverter.ToUInt64(verifyCrc, 0) != idxHeader.crc64)
+            {
+                DicConsole.DebugWriteLine("DiscImageChef format plugin", "Expected index CRC {0:X16} but got {1:X16}",
+                                          idxHeader.crc64, BitConverter.ToUInt64(verifyCrc, 0));
+                return false;
+            }
+
+            imageStream.Position -= structureBytes.Length;
+
+            List<IndexEntry> vrIndex = new List<IndexEntry>();
+            for(ushort i = 0; i < idxHeader.entries; i++)
+            {
+                IndexEntry entry = new IndexEntry();
+                structureBytes   = new byte[Marshal.SizeOf(entry)];
+                imageStream.Read(structureBytes, 0, structureBytes.Length);
+                structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(entry));
+                Marshal.Copy(structureBytes, 0, structurePointer, Marshal.SizeOf(entry));
+                entry = (IndexEntry)Marshal.PtrToStructure(structurePointer, typeof(IndexEntry));
+                Marshal.FreeHGlobal(structurePointer);
+                DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                          "Block type {0} with data type {1} is indexed to be at {2}", entry.blockType,
+                                          entry.dataType, entry.offset);
+                vrIndex.Add(entry);
+            }
+
+            const int VERIFY_SIZE = 1024 * 1024;
+
+            foreach(IndexEntry entry in vrIndex)
+            {
+                imageStream.Position = (long)entry.offset;
+                Crc64Context crcVerify;
+                ulong        readBytes;
+                byte[]       verifyBytes;
+
+                switch(entry.blockType)
+                {
+                    case BlockType.DataBlock:
+                        BlockHeader blockHeader = new BlockHeader();
+                        structureBytes          = new byte[Marshal.SizeOf(blockHeader)];
+                        imageStream.Read(structureBytes, 0, structureBytes.Length);
+                        structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(blockHeader));
+                        Marshal.Copy(structureBytes, 0, structurePointer, Marshal.SizeOf(blockHeader));
+                        blockHeader = (BlockHeader)Marshal.PtrToStructure(structurePointer, typeof(BlockHeader));
+                        Marshal.FreeHGlobal(structurePointer);
+
+                        crcVerify = new Crc64Context();
+                        crcVerify.Init();
+                        readBytes = 0;
+
+                        DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                  "Verifying data block type {0} at position {1}", entry.dataType,
+                                                  entry.offset);
+
+                        while(readBytes + VERIFY_SIZE < blockHeader.cmpLength)
+                        {
+                            verifyBytes = new byte[readBytes];
+                            imageStream.Read(verifyBytes, 0, verifyBytes.Length);
+                            crcVerify.Update(verifyBytes);
+                            readBytes += (ulong)verifyBytes.LongLength;
+                        }
+
+                        verifyBytes = new byte[blockHeader.cmpLength - readBytes];
+                        imageStream.Read(verifyBytes, 0, verifyBytes.Length);
+                        crcVerify.Update(verifyBytes);
+                        readBytes += (ulong)verifyBytes.LongLength;
+                        System.Console.WriteLine("Read {0} bytes of {1}", readBytes, blockHeader.cmpLength);
+
+                        verifyCrc = crcVerify.Final();
+
+                        if(BitConverter.ToUInt64(verifyCrc, 0) != blockHeader.cmpCrc64)
+                            DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                      "Expected block CRC {0:X16} but got {1:X16}",
+                                                      blockHeader.cmpCrc64, BitConverter.ToUInt64(verifyCrc, 0));
+
+                        break;
+                    case BlockType.DeDuplicationTable:
+                        DdtHeader ddtHeader = new DdtHeader();
+                        structureBytes      = new byte[Marshal.SizeOf(ddtHeader)];
+                        imageStream.Read(structureBytes, 0, structureBytes.Length);
+                        structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(ddtHeader));
+                        Marshal.Copy(structureBytes, 0, structurePointer, Marshal.SizeOf(ddtHeader));
+                        ddtHeader = (DdtHeader)Marshal.PtrToStructure(structurePointer, typeof(DdtHeader));
+                        Marshal.FreeHGlobal(structurePointer);
+
+                        crcVerify = new Crc64Context();
+                        crcVerify.Init();
+                        readBytes = 0;
+
+                        DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                  "Verifying deduplication table type {0} at position {1}",
+                                                  entry.dataType, entry.offset);
+
+                        while(readBytes + VERIFY_SIZE < ddtHeader.cmpLength)
+                        {
+                            verifyBytes = new byte[readBytes];
+                            imageStream.Read(verifyBytes, 0, verifyBytes.Length);
+                            crcVerify.Update(verifyBytes);
+                            readBytes += (ulong)verifyBytes.LongLength;
+                        }
+
+                        verifyBytes = new byte[ddtHeader.cmpLength - readBytes];
+                        imageStream.Read(verifyBytes, 0, verifyBytes.Length);
+                        crcVerify.Update(verifyBytes);
+
+                        verifyCrc = crcVerify.Final();
+
+                        if(BitConverter.ToUInt64(verifyCrc, 0) != ddtHeader.cmpCrc64)
+                        {
+                            DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                      "Expected block CRC {0:X16} but got {1:X16}", ddtHeader.cmpCrc64,
+                                                      BitConverter.ToUInt64(verifyCrc, 0));
+                            return false;
+                        }
+
+                        break;
+                    case BlockType.TracksBlock:
+                        TracksHeader trkHeader = new TracksHeader();
+                        structureBytes         = new byte[Marshal.SizeOf(trkHeader)];
+                        imageStream.Read(structureBytes, 0, structureBytes.Length);
+                        structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(trkHeader));
+                        Marshal.Copy(structureBytes, 0, structurePointer, Marshal.SizeOf(trkHeader));
+                        trkHeader = (TracksHeader)Marshal.PtrToStructure(structurePointer, typeof(IndexHeader));
+                        Marshal.FreeHGlobal(structurePointer);
+
+                        DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                  "Track block at {0} contains {1} entries", header.indexOffset,
+                                                  trkHeader.entries);
+
+                        structureBytes = new byte[Marshal.SizeOf(typeof(TrackEntry)) * trkHeader.entries];
+                        imageStream.Read(structureBytes, 0, structureBytes.Length);
+                        Crc64Context.Data(structureBytes, out verifyCrc);
+
+                        if(BitConverter.ToUInt64(verifyCrc, 0) != trkHeader.crc64)
+                        {
+                            DicConsole.DebugWriteLine("DiscImageChef format plugin",
+                                                      "Expected index CRC {0:X16} but got {1:X16}", trkHeader.crc64,
+                                                      BitConverter.ToUInt64(verifyCrc, 0));
+                            return false;
+                        }
+
+                        break;
+                    default:
+                        DicConsole.DebugWriteLine("DiscImageChef format plugin", "Ignored field type {0}",
+                                                  entry.blockType);
+                        break;
+                }
+            }
+
+            return true;
         }
 
         public IEnumerable<MediaTagType> SupportedMediaTags =>
@@ -1376,8 +1611,7 @@ namespace DiscImageChef.DiscImages
             {
                 ("sectors_per_block", typeof(uint),
                 "How many sectors to store per block (will be rounded to next power of two)"),
-                ("dictionary", typeof(uint),
+                ("dictionary", typeof(uint), "Size, in bytes, of the LZMA dictionary"),
-                 "Size, in bytes, of the LZMA dictionary"),
                 ("max_ddt_size", typeof(uint),
                 "Maximum size, in mebibytes, for in-memory DDT. If image needs a bigger one, it will be on-disk")
             };
@@ -1428,8 +1662,8 @@ namespace DiscImageChef.DiscImages
             else
             {
                 sectorsPerBlock = 4096;
-                dictionary = 1 << 25;
+                dictionary      = 1 << 25;
-                maxDdtSize = 256;
+                maxDdtSize      = 256;
             }
 
             if(!SupportedMediaTypes.Contains(mediaType))
@@ -1582,8 +1816,11 @@ namespace DiscImageChef.DiscImages
                 byte[] lzmaProperties     = compressedBlockStream.Properties;
                 compressedBlockStream.Close();
                 currentBlockHeader.cmpLength = (uint)blockStream.Length + LZMA_PROPERTIES_LENGTH;
-                Crc64Context.Data(blockStream.ToArray(), out byte[] cmpCrc64);
+                Crc64Context cmpCrc64Context = new Crc64Context();
-                currentBlockHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64, 0);
+                cmpCrc64Context.Init();
+                cmpCrc64Context.Update(lzmaProperties);
+                cmpCrc64Context.Update(blockStream.ToArray());
+                currentBlockHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64Context.Final(), 0);
 
                 index.Add(new IndexEntry
                 {
@@ -1920,8 +2157,11 @@ namespace DiscImageChef.DiscImages
                 byte[] lzmaProperties     = compressedBlockStream.Properties;
                 compressedBlockStream.Close();
                 currentBlockHeader.cmpLength = (uint)blockStream.Length + LZMA_PROPERTIES_LENGTH;
-                Crc64Context.Data(blockStream.ToArray(), out byte[] cmpCrc64);
+                Crc64Context cmpCrc64Context = new Crc64Context();
-                currentBlockHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64, 0);
+                cmpCrc64Context.Init();
+                cmpCrc64Context.Update(lzmaProperties);
+                cmpCrc64Context.Update(blockStream.ToArray());
+                currentBlockHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64Context.Final(), 0);
 
                 index.Add(new IndexEntry
                 {
@@ -2064,9 +2304,12 @@ namespace DiscImageChef.DiscImages
 
                 byte[] lzmaProperties = compressedBlockStream.Properties;
                 compressedBlockStream.Close();
-                ddtHeader.cmpLength = (uint)blockStream.Length + LZMA_PROPERTIES_LENGTH;
+                ddtHeader.cmpLength          = (uint)blockStream.Length + LZMA_PROPERTIES_LENGTH;
-                Crc64Context.Data(blockStream.ToArray(), out byte[] cmpCrc64);
+                Crc64Context cmpCrc64Context = new Crc64Context();
-                ddtHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64, 0);
+                cmpCrc64Context.Init();
+                cmpCrc64Context.Update(lzmaProperties);
+                cmpCrc64Context.Update(blockStream.ToArray());
+                ddtHeader.cmpCrc64 = BitConverter.ToUInt64(cmpCrc64Context.Final(), 0);
 
                 structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(ddtHeader));
                 structureBytes   = new byte[Marshal.SizeOf(ddtHeader)];