diff --git a/DiscImageChef.DiscImages/DiscImageChef.cs b/DiscImageChef.DiscImages/DiscImageChef.cs index d5fd1a52a..0898cc92f 100644 --- a/DiscImageChef.DiscImages/DiscImageChef.cs +++ b/DiscImageChef.DiscImages/DiscImageChef.cs @@ -117,6 +117,7 @@ namespace DiscImageChef.DiscImages /// smaller than 256. /// const int MIN_FLAKE_BLOCK = 256; + bool alreadyWrittenZero; /// Cache of uncompressed blocks. Dictionary blockCache; @@ -153,23 +154,29 @@ namespace DiscImageChef.DiscImages /// Index. List index; /// If set to true, the DDT entries are in-memory. - bool inMemoryDdt; + bool inMemoryDdt; + ulong lastWrittenBlock; /// LZMA stream. LzmaStream lzmaBlockStream; /// LZMA properties. LzmaEncoderProperties lzmaEncoderProperties; + Md5Context md5Provider; /// Cache of media tags. Dictionary mediaTags; /// If DDT is on-disk, this is the image stream offset at which it starts. long outMemoryDdtPosition; + bool rewinded; /// Cache for data that prefixes the user data on a sector (e.g. sync). byte[] sectorPrefix; /// Cache for data that goes side by side with user data (e.g. CompactDisc subchannel). byte[] sectorSubchannel; /// Cache for data that suffixes the user data on a sector (e.g. edc, ecc). - byte[] sectorSuffix; + byte[] sectorSuffix; + Sha1Context sha1Provider; + Sha256Context sha256Provider; /// Shift for calculating number of sectors in a block. - byte shift; + byte shift; + SpamSumContext spamsumProvider; /// Cache for bytes to write/rad on-disk. byte[] structureBytes; /// Cache for pointer for marshaling structures. @@ -180,6 +187,7 @@ namespace DiscImageChef.DiscImages Dictionary trackIsrcs; /// In-memory deduplication table ulong[] userDataDdt; + bool writingLong; public DiscImageChef() { @@ -1862,7 +1870,11 @@ namespace DiscImageChef.DiscImages "How many sectors to store per block (will be rounded to next power of two)"), ("dictionary", typeof(uint), "Size, in bytes, of the LZMA dictionary"), ("max_ddt_size", typeof(uint), - "Maximum size, in mebibytes, for in-memory DDT. If image needs a bigger one, it will be on-disk") + "Maximum size, in mebibytes, for in-memory DDT. If image needs a bigger one, it will be on-disk"), + ("md5", typeof(bool), "Calculate and store MD5 of image's user data"), + ("sha1", typeof(bool), "Calculate and store SHA1 of image's user data"), + ("sha256", typeof(bool), "Calculate and store SHA256 of image's user data"), + ("spamsum", typeof(bool), "Calculate and store SpamSum of image's user data") }; public IEnumerable KnownExtensions => new[] {".dicf"}; public bool IsWriting { get; private set; } @@ -1874,6 +1886,10 @@ namespace DiscImageChef.DiscImages uint sectorsPerBlock; uint dictionary; uint maxDdtSize; + bool doMd5; + bool doSha1; + bool doSha256; + bool doSpamsum; if(options != null) { @@ -1906,12 +1922,56 @@ namespace DiscImageChef.DiscImages } } else maxDdtSize = 256; + + if(options.TryGetValue("md5", out tmpValue)) + { + if(!bool.TryParse(tmpValue, out doMd5)) + { + ErrorMessage = "Invalid value for md5 option"; + return false; + } + } + else doMd5 = false; + + if(options.TryGetValue("sha1", out tmpValue)) + { + if(!bool.TryParse(tmpValue, out doSha1)) + { + ErrorMessage = "Invalid value for sha1 option"; + return false; + } + } + else doSha1 = false; + + if(options.TryGetValue("sha256", out tmpValue)) + { + if(!bool.TryParse(tmpValue, out doSha256)) + { + ErrorMessage = "Invalid value for sha256 option"; + return false; + } + } + else doSha256 = false; + + if(options.TryGetValue("spamsum", out tmpValue)) + { + if(!bool.TryParse(tmpValue, out doSpamsum)) + { + ErrorMessage = "Invalid value for spamsum option"; + return false; + } + } + else doSpamsum = false; } else { sectorsPerBlock = 4096; dictionary = 1 << 25; maxDdtSize = 256; + doMd5 = false; + doSha1 = false; + doSha256 = false; + doSpamsum = false; } // This really, cannot happen @@ -2001,6 +2061,12 @@ namespace DiscImageChef.DiscImages // If there exists an index, we are appending, so read index if(header.indexOffset > 0) { + // Can't calculate checksum of an appended image + md5Provider = null; + sha1Provider = null; + sha256Provider = null; + spamsumProvider = null; + imageStream.Position = (long)header.indexOffset; IndexHeader idxHeader = new IndexHeader(); structureBytes = new byte[Marshal.SizeOf(idxHeader)]; @@ -2034,6 +2100,9 @@ namespace DiscImageChef.DiscImages index.Add(entry); } + // Invalidate previous checksum block + index.RemoveAll(t => t.blockType == BlockType.ChecksumBlock && t.dataType == DataType.NoData); + bool foundUserDataDdt = false; foreach(IndexEntry entry in index) { @@ -2399,6 +2468,30 @@ namespace DiscImageChef.DiscImages imageStream.Position += (long)(sectors * sizeof(ulong)) - 1; imageStream.WriteByte(0); } + + if(doMd5) + { + md5Provider = new Md5Context(); + md5Provider.Init(); + } + + if(doSha1) + { + sha1Provider = new Sha1Context(); + sha1Provider.Init(); + } + + if(doSha256) + { + sha256Provider = new Sha256Context(); + sha256Provider.Init(); + } + + if(doSpamsum) + { + spamsumProvider = new SpamSumContext(); + spamsumProvider.Init(); + } } DicConsole.DebugWriteLine("DiscImageChef format plugin", "In memory DDT?: {0}", inMemoryDdt); @@ -2475,8 +2568,28 @@ namespace DiscImageChef.DiscImages return false; } + if((imageInfo.XmlMediaType != XmlMediaType.OpticalDisc || !writingLong) && !rewinded) + { + if(sectorAddress <= lastWrittenBlock && alreadyWrittenZero) + { + rewinded = true; + md5Provider = null; + sha1Provider = null; + sha256Provider = null; + spamsumProvider = null; + } + + md5Provider?.Update(data); + sha1Provider?.Update(data); + sha256Provider?.Update(data); + spamsumProvider?.Update(data); + lastWrittenBlock = sectorAddress; + } + byte[] hash = checksumProvider.ComputeHash(data); + if(sectorAddress == 0) alreadyWrittenZero = true; + if(deduplicationTable.TryGetValue(hash, out ulong pointer)) { SetDdtEntry(sectorAddress, pointer); @@ -2660,6 +2773,25 @@ namespace DiscImageChef.DiscImages return false; } + writingLong = true; + if(!rewinded) + { + if(sectorAddress <= lastWrittenBlock && alreadyWrittenZero) + { + rewinded = true; + md5Provider = null; + sha1Provider = null; + sha256Provider = null; + spamsumProvider = null; + } + + md5Provider?.Update(data); + sha1Provider?.Update(data); + sha256Provider?.Update(data); + spamsumProvider?.Update(data); + lastWrittenBlock = sectorAddress; + } + // Split raw cd sector data in prefix (sync, header), user data and suffix (edc, ecc p, ecc q) switch(track.TrackType) { @@ -3213,6 +3345,99 @@ namespace DiscImageChef.DiscImages index.Add(idxEntry); } + // If we have checksums, write it to disk + if(md5Provider != null || sha1Provider != null || sha256Provider != null || spamsumProvider != null) + { + MemoryStream chkMs = new MemoryStream(); + ChecksumHeader chkHeader = new ChecksumHeader {identifier = BlockType.ChecksumBlock}; + + if(md5Provider != null) + { + byte[] md5 = md5Provider.Final(); + ChecksumEntry md5Entry = + new ChecksumEntry {type = ChecksumAlgorithm.Md5, length = (uint)md5.Length}; + structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(md5Entry)); + structureBytes = new byte[Marshal.SizeOf(md5Entry)]; + Marshal.StructureToPtr(md5Entry, structurePointer, true); + Marshal.Copy(structurePointer, structureBytes, 0, structureBytes.Length); + Marshal.FreeHGlobal(structurePointer); + chkMs.Write(structureBytes, 0, structureBytes.Length); + chkMs.Write(md5, 0, md5.Length); + chkHeader.entries++; + } + + if(sha1Provider != null) + { + byte[] sha1 = sha1Provider.Final(); + ChecksumEntry sha1Entry = + new ChecksumEntry {type = ChecksumAlgorithm.Sha1, length = (uint)sha1.Length}; + structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(sha1Entry)); + structureBytes = new byte[Marshal.SizeOf(sha1Entry)]; + Marshal.StructureToPtr(sha1Entry, structurePointer, true); + Marshal.Copy(structurePointer, structureBytes, 0, structureBytes.Length); + Marshal.FreeHGlobal(structurePointer); + chkMs.Write(structureBytes, 0, structureBytes.Length); + chkMs.Write(sha1, 0, sha1.Length); + chkHeader.entries++; + } + + if(sha256Provider != null) + { + byte[] sha256 = sha256Provider.Final(); + ChecksumEntry sha256Entry = + new ChecksumEntry {type = ChecksumAlgorithm.Sha256, length = (uint)sha256.Length}; + structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(sha256Entry)); + structureBytes = new byte[Marshal.SizeOf(sha256Entry)]; + Marshal.StructureToPtr(sha256Entry, structurePointer, true); + Marshal.Copy(structurePointer, structureBytes, 0, structureBytes.Length); + Marshal.FreeHGlobal(structurePointer); + chkMs.Write(structureBytes, 0, structureBytes.Length); + chkMs.Write(sha256, 0, sha256.Length); + chkHeader.entries++; + } + + if(spamsumProvider != null) + { + byte[] spamsum = Encoding.ASCII.GetBytes(spamsumProvider.End()); + ChecksumEntry spamsumEntry = + new ChecksumEntry {type = ChecksumAlgorithm.SpamSum, length = (uint)spamsum.Length}; + structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(spamsumEntry)); + structureBytes = new byte[Marshal.SizeOf(spamsumEntry)]; + Marshal.StructureToPtr(spamsumEntry, structurePointer, true); + Marshal.Copy(structurePointer, structureBytes, 0, structureBytes.Length); + Marshal.FreeHGlobal(structurePointer); + chkMs.Write(structureBytes, 0, structureBytes.Length); + chkMs.Write(spamsum, 0, spamsum.Length); + chkHeader.entries++; + } + + if(chkHeader.entries > 0) + { + chkHeader.length = (uint)chkMs.Length; + idxEntry = new IndexEntry + { + blockType = BlockType.ChecksumBlock, + dataType = DataType.NoData, + offset = (ulong)imageStream.Position + }; + + DicConsole.DebugWriteLine("DiscImageChef format plugin", "Writing checksum block to position {0}", + idxEntry.offset); + + structurePointer = Marshal.AllocHGlobal(Marshal.SizeOf(chkHeader)); + structureBytes = new byte[Marshal.SizeOf(chkHeader)]; + Marshal.StructureToPtr(chkHeader, structurePointer, true); + Marshal.Copy(structurePointer, structureBytes, 0, structureBytes.Length); + Marshal.FreeHGlobal(structurePointer); + imageStream.Write(structureBytes, 0, structureBytes.Length); + imageStream.Write(chkMs.ToArray(), 0, (int)chkMs.Length); + + index.RemoveAll(t => t.blockType == BlockType.ChecksumBlock && t.dataType == DataType.NoData); + + index.Add(idxEntry); + } + } + // If the DDT is in-memory, write it to disk if(inMemoryDdt) { @@ -4506,7 +4731,7 @@ namespace DiscImageChef.DiscImages TracksBlock = 0x534B5254, /// Block containing CICM XML metadata CicmBlock = 0x4D434943, - /// TODO: Block containing contents checksums + /// Block containing contents checksums ChecksumBlock = 0x4D534B43, /// TODO: Block containing data position measurements DataPositionMeasurementBlock = 0x2A4D5044, @@ -4779,5 +5004,39 @@ namespace DiscImageChef.DiscImages public ulong start; public ulong end; } + + /// + /// Checksum block, contains a checksum of all user data sectors (except for optical discs that is 2352 bytes raw + /// sector if available + /// + [StructLayout(LayoutKind.Sequential, Pack = 1)] + struct ChecksumHeader + { + /// Identifier, + public BlockType identifier; + /// Length in bytes of the block + public uint length; + /// How many checksums follow + public byte entries; + } + + enum ChecksumAlgorithm : byte + { + Invalid = 0, + Md5 = 1, + Sha1 = 2, + Sha256 = 3, + SpamSum = 4 + } + + /// Checksum entry, followed by checksum data itself + [StructLayout(LayoutKind.Sequential, Pack = 1)] + struct ChecksumEntry + { + /// Checksum algorithm + public ChecksumAlgorithm type; + /// Length in bytes of checksum that follows this structure + public uint length; + } } } \ No newline at end of file diff --git a/templates/dicformat.bt b/templates/dicformat.bt index 0eac4e2c8..aabc004ad 100644 --- a/templates/dicformat.bt +++ b/templates/dicformat.bt @@ -645,6 +645,15 @@ enum TrackType CdMode2Form2 = 5 }; +enum ChecksumAlgorithm +{ + Invalid = 0, + Md5 = 1, + Sha1 = 2, + Sha256 = 3, + SpamSum = 4 +}; + typedef struct { char identifier[8]; @@ -803,6 +812,24 @@ typedef struct DumpHardwareEntry dumpHardware[entries]; } DumpHardwareHeader; +typedef struct +{ + ChecksumAlgorithm type; + uint length; + if(type == 4) + char checksum[length] ; + else + byte checksum[length]; +} ChecksumEntry ; + +typedef struct +{ + BlockType identifier; + uint length; + byte entries; + ChecksumEntry checksums[entries]; +} ChecksumHeader; + LittleEndian(); local int i; @@ -899,5 +926,8 @@ for(i = 0; i < index.entries; i++) case 0x2A504D44: DumpHardwareHeader dumpHardware; break; + case 0x4D534B43: + ChecksumHeader checksum; + break; } } \ No newline at end of file