diff --git a/SabreTools.Library/DatFiles/AttractMode.cs b/SabreTools.Library/DatFiles/AttractMode.cs index 7e2f379e..258b361c 100644 --- a/SabreTools.Library/DatFiles/AttractMode.cs +++ b/SabreTools.Library/DatFiles/AttractMode.cs @@ -90,6 +90,7 @@ namespace SabreTools.Library.DatFiles Size = Constants.SizeZero, CRC = Constants.CRCZero, MD5 = Constants.MD5Zero, + RIPEMD160 = Constants.RIPEMD160Zero, SHA1 = Constants.SHA1Zero, ItemStatus = ItemStatus.None, diff --git a/SabreTools.Library/DatFiles/ClrMamePro.cs b/SabreTools.Library/DatFiles/ClrMamePro.cs index de359106..63830265 100644 --- a/SabreTools.Library/DatFiles/ClrMamePro.cs +++ b/SabreTools.Library/DatFiles/ClrMamePro.cs @@ -342,6 +342,7 @@ namespace SabreTools.Library.DatFiles && linegc[i] != "date" && linegc[i] != "crc" && linegc[i] != "md5" + && linegc[i] != "ripemd160" && linegc[i] != "sha1" && linegc[i] != "sha256" && linegc[i] != "sha384" @@ -384,6 +385,12 @@ namespace SabreTools.Library.DatFiles ((Rom)item).MD5 = linegc[++i].Replace("\"", "").ToLowerInvariant(); } + // Get the RIPEMD160 from the next part + else if (linegc[i] == "ripemd160") + { + ((Rom)item).RIPEMD160 = linegc[++i].Replace("\"", "").ToLowerInvariant(); + } + // Get the SHA1 from the next part else if (linegc[i] == "sha1") { @@ -481,6 +488,18 @@ namespace SabreTools.Library.DatFiles ((Disk)item).MD5 = Utilities.CleanHashData(quoteless, Constants.MD5Length); } break; + case "ripemd160": + if (item.ItemType == ItemType.Rom) + { + quoteless = linegc[++i].Replace("\"", ""); + ((Rom)item).RIPEMD160 = Utilities.CleanHashData(quoteless, Constants.RIPEMD160Length); + } + else if (item.ItemType == ItemType.Disk) + { + quoteless = linegc[++i].Replace("\"", ""); + ((Disk)item).RIPEMD160 = Utilities.CleanHashData(quoteless, Constants.RIPEMD160Length); + } + break; case "sha1": if (item.ItemType == ItemType.Rom) { @@ -738,6 +757,7 @@ namespace SabreTools.Library.DatFiles ((Rom)rom).Size = Constants.SizeZero; ((Rom)rom).CRC = ((Rom)rom).CRC == "null" ? Constants.CRCZero : null; ((Rom)rom).MD5 = ((Rom)rom).MD5 == "null" ? Constants.MD5Zero : null; + ((Rom)rom).RIPEMD160 = ((Rom)rom).RIPEMD160 == "null" ? Constants.RIPEMD160Zero : null; ((Rom)rom).SHA1 = ((Rom)rom).SHA1 == "null" ? Constants.SHA1Zero : null; ((Rom)rom).SHA256 = ((Rom)rom).SHA256 == "null" ? Constants.SHA256Zero : null; ((Rom)rom).SHA384 = ((Rom)rom).SHA384 == "null" ? Constants.SHA384Zero : null; @@ -911,6 +931,7 @@ namespace SabreTools.Library.DatFiles case ItemType.Disk: state += "\tdisk ( name \"" + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") + "\"" + (!ExcludeFields[(int)Field.MD5] && !String.IsNullOrWhiteSpace(((Disk)rom).MD5) ? " md5 " + ((Disk)rom).MD5.ToLowerInvariant() : "") + + (!ExcludeFields[(int)Field.RIPEMD160] && !String.IsNullOrWhiteSpace(((Disk)rom).RIPEMD160) ? " ripemd160 " + ((Disk)rom).RIPEMD160.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA1] && !String.IsNullOrWhiteSpace(((Disk)rom).SHA1) ? " sha1 " + ((Disk)rom).SHA1.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA256] && !String.IsNullOrWhiteSpace(((Disk)rom).SHA256) ? " sha256 " + ((Disk)rom).SHA256.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA384] && !String.IsNullOrWhiteSpace(((Disk)rom).SHA384) ? " sha384 " + ((Disk)rom).SHA384.ToLowerInvariant() : "") @@ -933,6 +954,7 @@ namespace SabreTools.Library.DatFiles + (!ExcludeFields[(int)Field.Size] && ((Rom)rom).Size != -1 ? " size " + ((Rom)rom).Size : "") + (!ExcludeFields[(int)Field.CRC] && !String.IsNullOrWhiteSpace(((Rom)rom).CRC) ? " crc " + ((Rom)rom).CRC.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.MD5] && !String.IsNullOrWhiteSpace(((Rom)rom).MD5) ? " md5 " + ((Rom)rom).MD5.ToLowerInvariant() : "") + + (!ExcludeFields[(int)Field.RIPEMD160] && !String.IsNullOrWhiteSpace(((Rom)rom).RIPEMD160) ? " ripemd160 " + ((Rom)rom).RIPEMD160.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA1] && !String.IsNullOrWhiteSpace(((Rom)rom).SHA1) ? " sha1 " + ((Rom)rom).SHA1.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA256] && !String.IsNullOrWhiteSpace(((Rom)rom).SHA256) ? " sha256 " + ((Rom)rom).SHA256.ToLowerInvariant() : "") + (!ExcludeFields[(int)Field.SHA384] && !String.IsNullOrWhiteSpace(((Rom)rom).SHA384) ? " sha384 " + ((Rom)rom).SHA384.ToLowerInvariant() : "") diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index c7daac21..3fb8a93b 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -815,6 +815,23 @@ namespace SabreTools.Library.DatFiles } } + /// + /// Number of items with a RIPEMD160 hash + /// + public long RIPEMD160Count + { + get + { + EnsureDatStats(); + return _datStats.RIPEMD160Count; + } + private set + { + EnsureDatStats(); + _datStats.RIPEMD160Count = value; + } + } + /// /// Number of items with a SHA-1 hash /// @@ -1376,6 +1393,12 @@ namespace SabreTools.Library.DatFiles BucketBy(SortedBy.SHA1, deduperoms, lower, norename); } + // If all items are supposed to have a SHA-1, we sort by that + else if (RomCount + DiskCount - NodumpCount == RIPEMD160Count) + { + BucketBy(SortedBy.RIPEMD160, deduperoms, lower, norename); + } + // If all items are supposed to have a MD5, we sort by that else if (RomCount + DiskCount - NodumpCount == MD5Count) { @@ -1702,6 +1725,10 @@ namespace SabreTools.Library.DatFiles { newRomItem.MD5 = ((Rom)dupes[0]).MD5; } + if (String.IsNullOrEmpty(newRomItem.RIPEMD160) && !String.IsNullOrEmpty(((Rom)dupes[0]).RIPEMD160)) + { + newRomItem.RIPEMD160 = ((Rom)dupes[0]).RIPEMD160; + } if (String.IsNullOrEmpty(newRomItem.SHA1) && !String.IsNullOrEmpty(((Rom)dupes[0]).SHA1)) { newRomItem.SHA1 = ((Rom)dupes[0]).SHA1; @@ -1728,6 +1755,10 @@ namespace SabreTools.Library.DatFiles { newDiskItem.MD5 = ((Disk)dupes[0]).MD5; } + if (String.IsNullOrEmpty(newDiskItem.RIPEMD160) && !String.IsNullOrEmpty(((Disk)dupes[0]).RIPEMD160)) + { + newDiskItem.RIPEMD160 = ((Disk)dupes[0]).RIPEMD160; + } if (String.IsNullOrEmpty(newDiskItem.SHA1) && !String.IsNullOrEmpty(((Disk)dupes[0]).SHA1)) { newDiskItem.SHA1 = ((Disk)dupes[0]).SHA1; @@ -3129,6 +3160,7 @@ namespace SabreTools.Library.DatFiles // Sanitize the hashes from null, hex sizes, and "true blank" strings itemRom.CRC = Utilities.CleanHashData(itemRom.CRC, Constants.CRCLength); itemRom.MD5 = Utilities.CleanHashData(itemRom.MD5, Constants.MD5Length); + itemRom.RIPEMD160 = Utilities.CleanHashData(itemRom.RIPEMD160, Constants.RIPEMD160Length); itemRom.SHA1 = Utilities.CleanHashData(itemRom.SHA1, Constants.SHA1Length); itemRom.SHA256 = Utilities.CleanHashData(itemRom.SHA256, Constants.SHA256Length); itemRom.SHA384 = Utilities.CleanHashData(itemRom.SHA384, Constants.SHA384Length); @@ -3138,6 +3170,7 @@ namespace SabreTools.Library.DatFiles if (itemRom.Size == -1 && String.IsNullOrWhiteSpace(itemRom.CRC) && String.IsNullOrWhiteSpace(itemRom.MD5) + && String.IsNullOrWhiteSpace(itemRom.RIPEMD160) && !String.IsNullOrWhiteSpace(itemRom.SHA1) && String.IsNullOrWhiteSpace(itemRom.SHA256) && String.IsNullOrWhiteSpace(itemRom.SHA384) @@ -3151,6 +3184,7 @@ namespace SabreTools.Library.DatFiles else if ((itemRom.Size == 0 || itemRom.Size == -1) && ((itemRom.CRC == Constants.CRCZero || String.IsNullOrWhiteSpace(itemRom.CRC)) || itemRom.MD5 == Constants.MD5Zero + || itemRom.RIPEMD160 == Constants.RIPEMD160Zero || itemRom.SHA1 == Constants.SHA1Zero || itemRom.SHA256 == Constants.SHA256Zero || itemRom.SHA384 == Constants.SHA384Zero @@ -3160,12 +3194,14 @@ namespace SabreTools.Library.DatFiles itemRom.Size = Constants.SizeZero; itemRom.CRC = Constants.CRCZero; itemRom.MD5 = Constants.MD5Zero; + itemRom.RIPEMD160 = null; + //itemRom.RIPEMD160 = Constants.RIPEMD160Zero; itemRom.SHA1 = Constants.SHA1Zero; itemRom.SHA256 = null; - itemRom.SHA384 = null; - itemRom.SHA512 = null; //itemRom.SHA256 = Constants.SHA256Zero; + itemRom.SHA384 = null; //itemRom.SHA384 = Constants.SHA384Zero; + itemRom.SHA512 = null; //itemRom.SHA512 = Constants.SHA512Zero; } // If the file has no size and it's not the above case, skip and log @@ -3179,6 +3215,7 @@ namespace SabreTools.Library.DatFiles && itemRom.Size > 0 && String.IsNullOrWhiteSpace(itemRom.CRC) && String.IsNullOrWhiteSpace(itemRom.MD5) + && String.IsNullOrWhiteSpace(itemRom.RIPEMD160) && String.IsNullOrWhiteSpace(itemRom.SHA1) && String.IsNullOrWhiteSpace(itemRom.SHA256) && String.IsNullOrWhiteSpace(itemRom.SHA384) @@ -3196,6 +3233,7 @@ namespace SabreTools.Library.DatFiles // Sanitize the hashes from null, hex sizes, and "true blank" strings itemDisk.MD5 = Utilities.CleanHashData(itemDisk.MD5, Constants.MD5Length); + itemDisk.RIPEMD160 = Utilities.CleanHashData(itemDisk.RIPEMD160, Constants.RIPEMD160Length); itemDisk.SHA1 = Utilities.CleanHashData(itemDisk.SHA1, Constants.SHA1Length); itemDisk.SHA256 = Utilities.CleanHashData(itemDisk.SHA256, Constants.SHA256Length); itemDisk.SHA384 = Utilities.CleanHashData(itemDisk.SHA384, Constants.SHA384Length); @@ -3204,6 +3242,7 @@ namespace SabreTools.Library.DatFiles // If the file has aboslutely no hashes, skip and log if (itemDisk.ItemStatus != ItemStatus.Nodump && String.IsNullOrWhiteSpace(itemDisk.MD5) + && String.IsNullOrWhiteSpace(itemDisk.RIPEMD160) && String.IsNullOrWhiteSpace(itemDisk.SHA1) && String.IsNullOrWhiteSpace(itemDisk.SHA256) && String.IsNullOrWhiteSpace(itemDisk.SHA384) @@ -4866,6 +4905,27 @@ namespace SabreTools.Library.DatFiles DatFormat = this.DatFormat, DedupeRoms = this.DedupeRoms, }; + DatFile ripemd160 = new DatFile + { + FileName = this.FileName + " (RIPEMD160)", + Name = this.Name + " (RIPEMD160)", + Description = this.Description + " (RIPEMD160)", + Category = this.Category, + Version = this.Version, + Date = this.Date, + Author = this.Author, + Email = this.Email, + Homepage = this.Homepage, + Url = this.Url, + Comment = this.Comment, + Header = this.Header, + Type = this.Type, + ForceMerging = this.ForceMerging, + ForceNodump = this.ForceNodump, + ForcePacking = this.ForcePacking, + DatFormat = this.DatFormat, + DedupeRoms = this.DedupeRoms, + }; DatFile md5 = new DatFile { FileName = this.FileName + " (MD5)", @@ -4973,13 +5033,19 @@ namespace SabreTools.Library.DatFiles { sha1.Add(key, item); } - // If the file has no SHA-1 but has an MD5 + // If the file has a SHA-1 + else if ((item.ItemType == ItemType.Rom && !String.IsNullOrWhiteSpace(((Rom)item).RIPEMD160)) + || (item.ItemType == ItemType.Disk && !String.IsNullOrWhiteSpace(((Disk)item).RIPEMD160))) + { + ripemd160.Add(key, item); + } + // If the file has an MD5 else if ((item.ItemType == ItemType.Rom && !String.IsNullOrWhiteSpace(((Rom)item).MD5)) || (item.ItemType == ItemType.Disk && !String.IsNullOrWhiteSpace(((Disk)item).MD5))) { md5.Add(key, item); } - // If the file has no MD5 but a CRC + // If the file has a CRC else if ((item.ItemType == ItemType.Rom && !String.IsNullOrWhiteSpace(((Rom)item).CRC))) { crc.Add(key, item); @@ -4999,6 +5065,7 @@ namespace SabreTools.Library.DatFiles success &= sha384.Write(outDir); success &= sha256.Write(outDir); success &= sha1.Write(outDir); + success &= ripemd160.Write(outDir); success &= md5.Write(outDir); success &= crc.Write(outDir); @@ -5333,6 +5400,7 @@ namespace SabreTools.Library.DatFiles Disks found: " + DiskCount + @" Roms with CRC: " + CRCCount + @" Roms with MD5: " + MD5Count + @" + Roms with RIPEMD160: " + RIPEMD160Count + @" Roms with SHA-1: " + SHA1Count + @" Roms with SHA-256: " + SHA256Count + @" Roms with SHA-384: " + SHA384Count + @" @@ -5670,6 +5738,12 @@ namespace SabreTools.Library.DatFiles outfileNames.Add(DatFormat.RedumpMD5, CreateOutfileNamesHelper(outDir, ".md5", overwrite)); }; + // Redump RIPEMD160 + if ((DatFormat & DatFormat.RedumpRIPEMD160) != 0) + { + outfileNames.Add(DatFormat.RedumpRIPEMD160, CreateOutfileNamesHelper(outDir, ".ripemd160", overwrite)); + }; + // Redump SFV if ((DatFormat & DatFormat.RedumpSFV) != 0) { @@ -5895,6 +5969,7 @@ namespace SabreTools.Library.DatFiles publisher = item.Publisher, crc = string.Empty, md5 = string.Empty, + ripemd160 = string.Empty, sha1 = string.Empty, sha256 = string.Empty, sha384 = string.Empty, @@ -5917,6 +5992,7 @@ namespace SabreTools.Library.DatFiles { crc = ((Rom)item).CRC; md5 = ((Rom)item).MD5; + ripemd160 = ((Rom)item).RIPEMD160; sha1 = ((Rom)item).SHA1; sha256 = ((Rom)item).SHA256; sha384 = ((Rom)item).SHA384; @@ -5926,6 +6002,7 @@ namespace SabreTools.Library.DatFiles else if (item.ItemType == ItemType.Disk) { md5 = ((Disk)item).MD5; + ripemd160 = ((Disk)item).RIPEMD160; sha1 = ((Disk)item).SHA1; sha256 = ((Disk)item).SHA256; sha384 = ((Disk)item).SHA384; @@ -5941,6 +6018,7 @@ namespace SabreTools.Library.DatFiles .Replace("%publisher%", publisher) .Replace("%crc%", crc) .Replace("%md5%", md5) + .Replace("%ripemd160%", ripemd160) .Replace("%sha1%", sha1) .Replace("%sha256%", sha256) .Replace("%sha384%", sha384) diff --git a/SabreTools.Library/DatFiles/DatStats.cs b/SabreTools.Library/DatFiles/DatStats.cs index b8c46aa4..b40e0391 100644 --- a/SabreTools.Library/DatFiles/DatStats.cs +++ b/SabreTools.Library/DatFiles/DatStats.cs @@ -81,6 +81,11 @@ namespace SabreTools.Library.DatFiles /// public long MD5Count { get; set; } = 0; + /// + /// Number of items with a RIPEMD160 hash + /// + public long RIPEMD160Count { get; set; } = 0; + /// /// Number of items with a SHA-1 hash /// @@ -151,6 +156,7 @@ namespace SabreTools.Library.DatFiles if (((Disk)item).ItemStatus != ItemStatus.Nodump) { this.MD5Count += (String.IsNullOrWhiteSpace(((Disk)item).MD5) ? 0 : 1); + this.RIPEMD160Count += (String.IsNullOrWhiteSpace(((Disk)item).RIPEMD160) ? 0 : 1); this.SHA1Count += (String.IsNullOrWhiteSpace(((Disk)item).SHA1) ? 0 : 1); this.SHA256Count += (String.IsNullOrWhiteSpace(((Disk)item).SHA256) ? 0 : 1); this.SHA384Count += (String.IsNullOrWhiteSpace(((Disk)item).SHA384) ? 0 : 1); @@ -172,6 +178,7 @@ namespace SabreTools.Library.DatFiles this.TotalSize += ((Rom)item).Size; this.CRCCount += (String.IsNullOrWhiteSpace(((Rom)item).CRC) ? 0 : 1); this.MD5Count += (String.IsNullOrWhiteSpace(((Rom)item).MD5) ? 0 : 1); + this.RIPEMD160Count += (String.IsNullOrWhiteSpace(((Rom)item).RIPEMD160) ? 0 : 1); this.SHA1Count += (String.IsNullOrWhiteSpace(((Rom)item).SHA1) ? 0 : 1); this.SHA256Count += (String.IsNullOrWhiteSpace(((Rom)item).SHA256) ? 0 : 1); this.SHA384Count += (String.IsNullOrWhiteSpace(((Rom)item).SHA384) ? 0 : 1); @@ -212,6 +219,7 @@ namespace SabreTools.Library.DatFiles // Individual hash counts this.CRCCount += stats.CRCCount; this.MD5Count += stats.MD5Count; + this.RIPEMD160Count += stats.RIPEMD160Count; this.SHA1Count += stats.SHA1Count; this.SHA256Count += stats.SHA256Count; this.SHA384Count += stats.SHA384Count; @@ -250,6 +258,7 @@ namespace SabreTools.Library.DatFiles if (((Disk)item).ItemStatus != ItemStatus.Nodump) { this.MD5Count -= (String.IsNullOrWhiteSpace(((Disk)item).MD5) ? 0 : 1); + this.RIPEMD160Count -= (String.IsNullOrWhiteSpace(((Disk)item).RIPEMD160) ? 0 : 1); this.SHA1Count -= (String.IsNullOrWhiteSpace(((Disk)item).SHA1) ? 0 : 1); this.SHA256Count -= (String.IsNullOrWhiteSpace(((Disk)item).SHA256) ? 0 : 1); this.SHA384Count -= (String.IsNullOrWhiteSpace(((Disk)item).SHA384) ? 0 : 1); @@ -271,6 +280,7 @@ namespace SabreTools.Library.DatFiles this.TotalSize -= ((Rom)item).Size; this.CRCCount -= (String.IsNullOrWhiteSpace(((Rom)item).CRC) ? 0 : 1); this.MD5Count -= (String.IsNullOrWhiteSpace(((Rom)item).MD5) ? 0 : 1); + this.RIPEMD160Count -= (String.IsNullOrWhiteSpace(((Rom)item).RIPEMD160) ? 0 : 1); this.SHA1Count -= (String.IsNullOrWhiteSpace(((Rom)item).SHA1) ? 0 : 1); this.SHA256Count -= (String.IsNullOrWhiteSpace(((Rom)item).SHA256) ? 0 : 1); this.SHA384Count -= (String.IsNullOrWhiteSpace(((Rom)item).SHA384) ? 0 : 1); @@ -309,6 +319,7 @@ namespace SabreTools.Library.DatFiles this.CRCCount = 0; this.MD5Count = 0; + this.RIPEMD160Count = 0; this.SHA1Count = 0; this.SHA256Count = 0; this.SHA384Count = 0; diff --git a/SabreTools.Library/DatFiles/DosCenter.cs b/SabreTools.Library/DatFiles/DosCenter.cs index 2c1b95e4..6afc91cc 100644 --- a/SabreTools.Library/DatFiles/DosCenter.cs +++ b/SabreTools.Library/DatFiles/DosCenter.cs @@ -133,6 +133,7 @@ namespace SabreTools.Library.DatFiles ((Rom)rom).Size = Constants.SizeZero; ((Rom)rom).CRC = ((Rom)rom).CRC == "null" ? Constants.CRCZero : null; ((Rom)rom).MD5 = ((Rom)rom).MD5 == "null" ? Constants.MD5Zero : null; + ((Rom)rom).RIPEMD160 = ((Rom)rom).RIPEMD160 == "null" ? Constants.RIPEMD160Zero : null; ((Rom)rom).SHA1 = ((Rom)rom).SHA1 == "null" ? Constants.SHA1Zero : null; ((Rom)rom).SHA256 = ((Rom)rom).SHA256 == "null" ? Constants.SHA256Zero : null; ((Rom)rom).SHA384 = ((Rom)rom).SHA384 == "null" ? Constants.SHA384Zero : null; diff --git a/SabreTools.Library/DatFiles/EverdriveSmdb.cs b/SabreTools.Library/DatFiles/EverdriveSmdb.cs index f6c27ab7..4d9b2f7d 100644 --- a/SabreTools.Library/DatFiles/EverdriveSmdb.cs +++ b/SabreTools.Library/DatFiles/EverdriveSmdb.cs @@ -76,10 +76,10 @@ namespace SabreTools.Library.DatFiles { Name = gameinfo[1].Substring(fullname[0].Length + 1), Size = -1, // No size provided, but we don't want the size being 0 - CRC = Utilities.CleanHashData(gameinfo[4], 8), - MD5 = Utilities.CleanHashData(gameinfo[3], 32), - SHA1 = Utilities.CleanHashData(gameinfo[2], 40), - SHA256 = Utilities.CleanHashData(gameinfo[0], 64), + CRC = Utilities.CleanHashData(gameinfo[4], Constants.CRCLength), + MD5 = Utilities.CleanHashData(gameinfo[3], Constants.MD5Length), + SHA1 = Utilities.CleanHashData(gameinfo[2], Constants.SHA1Length), + SHA256 = Utilities.CleanHashData(gameinfo[0], Constants.SHA256Length), ItemStatus = ItemStatus.None, MachineName = fullname[0], diff --git a/SabreTools.Library/DatFiles/Filter.cs b/SabreTools.Library/DatFiles/Filter.cs index 65d40ca8..90e9c70c 100644 --- a/SabreTools.Library/DatFiles/Filter.cs +++ b/SabreTools.Library/DatFiles/Filter.cs @@ -48,6 +48,11 @@ namespace SabreTools.Library.DatFiles /// public FilterItem MD5 { get; set; } = new FilterItem(); + /// + /// Include or exclude RIPEMD160 hashes + /// + public FilterItem RIPEMD160 { get; set; } = new FilterItem(); + /// /// Include or exclude SHA-1 hashes /// @@ -224,6 +229,12 @@ namespace SabreTools.Library.DatFiles if (this.MD5.MatchesNegativeSet(rom.MD5) == true) return false; + // Filter on RIPEMD160 + if (this.RIPEMD160.MatchesPositiveSet(rom.RIPEMD160) == false) + return false; + if (this.RIPEMD160.MatchesNegativeSet(rom.RIPEMD160) == true) + return false; + // Filter on SHA-1 if (this.SHA1.MatchesPositiveSet(rom.SHA1) == false) return false; @@ -264,6 +275,12 @@ namespace SabreTools.Library.DatFiles if (this.MD5.MatchesNegativeSet(rom.MD5) == true) return false; + // Filter on RIPEMD160 + if (this.RIPEMD160.MatchesPositiveSet(rom.RIPEMD160) == false) + return false; + if (this.RIPEMD160.MatchesNegativeSet(rom.RIPEMD160) == true) + return false; + // Filter on SHA-1 if (this.SHA1.MatchesPositiveSet(rom.SHA1) == false) return false; diff --git a/SabreTools.Library/DatFiles/Hashfile.cs b/SabreTools.Library/DatFiles/Hashfile.cs index 8c11d9bb..ea39a8af 100644 --- a/SabreTools.Library/DatFiles/Hashfile.cs +++ b/SabreTools.Library/DatFiles/Hashfile.cs @@ -89,6 +89,7 @@ namespace SabreTools.Library.DatFiles Size = -1, CRC = ((_hash & Hash.CRC) != 0 ? Utilities.CleanHashData(hash, Constants.CRCLength) : null), MD5 = ((_hash & Hash.MD5) != 0 ? Utilities.CleanHashData(hash, Constants.MD5Length) : null), + RIPEMD160 = ((_hash & Hash.RIPEMD160) != 0 ? Utilities.CleanHashData(hash, Constants.RIPEMD160Length) : null), SHA1 = ((_hash & Hash.SHA1) != 0 ? Utilities.CleanHashData(hash, Constants.SHA1Length) : null), SHA256 = ((_hash & Hash.SHA256) != 0 ? Utilities.CleanHashData(hash, Constants.SHA256Length) : null), SHA384 = ((_hash & Hash.SHA384) != 0 ? Utilities.CleanHashData(hash, Constants.SHA384Length) : null), @@ -204,6 +205,14 @@ namespace SabreTools.Library.DatFiles switch (_hash) { + case Hash.CRC: + if (rom.ItemType == ItemType.Rom) + { + state += (!ExcludeFields[(int)Field.MachineName] && GameName ? rom.MachineName + Path.DirectorySeparatorChar : "") + + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") + + " " + (!ExcludeFields[(int)Field.CRC] ? ((Rom)rom).CRC : "") + "\n"; + } + break; case Hash.MD5: if (rom.ItemType == ItemType.Rom) { @@ -218,12 +227,18 @@ namespace SabreTools.Library.DatFiles + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") + "\n"; } break; - case Hash.CRC: + case Hash.RIPEMD160: if (rom.ItemType == ItemType.Rom) { - state += (!ExcludeFields[(int)Field.MachineName] && GameName ? rom.MachineName + Path.DirectorySeparatorChar : "") - + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") - + " " + (!ExcludeFields[(int)Field.CRC] ? ((Rom)rom).CRC : "") + "\n"; + state += (!ExcludeFields[(int)Field.RIPEMD160] ? ((Rom)rom).RIPEMD160 : "") + + " *" + (!ExcludeFields[(int)Field.MachineName] && GameName ? rom.MachineName + Path.DirectorySeparatorChar : "") + + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") + "\n"; + } + else if (rom.ItemType == ItemType.Disk) + { + state += (!ExcludeFields[(int)Field.RIPEMD160] ? ((Disk)rom).RIPEMD160 : "") + + " *" + (!ExcludeFields[(int)Field.MachineName] && GameName ? rom.MachineName + Path.DirectorySeparatorChar : "") + + (!ExcludeFields[(int)Field.Name] ? rom.Name : "") + "\n"; } break; case Hash.SHA1: diff --git a/SabreTools.Library/DatFiles/Listxml.cs b/SabreTools.Library/DatFiles/Listxml.cs index 4b318808..038e8a6d 100644 --- a/SabreTools.Library/DatFiles/Listxml.cs +++ b/SabreTools.Library/DatFiles/Listxml.cs @@ -241,6 +241,7 @@ namespace SabreTools.Library.DatFiles Size = Utilities.GetSize(reader.GetAttribute("size")), CRC = Utilities.CleanHashData(reader.GetAttribute("crc"), Constants.CRCLength), MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length), + RIPEMD160 = Utilities.CleanHashData(reader.GetAttribute("ripemd160"), Constants.SHA1Length), SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length), SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length), SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length), @@ -270,6 +271,7 @@ namespace SabreTools.Library.DatFiles { Name = reader.GetAttribute("name"), MD5 = Utilities.CleanHashData(reader.GetAttribute("md5"), Constants.MD5Length), + RIPEMD160 = Utilities.CleanHashData(reader.GetAttribute("ripemd160"), Constants.SHA1Length), SHA1 = Utilities.CleanHashData(reader.GetAttribute("sha1"), Constants.SHA1Length), SHA256 = Utilities.CleanHashData(reader.GetAttribute("sha256"), Constants.SHA256Length), SHA384 = Utilities.CleanHashData(reader.GetAttribute("sha384"), Constants.SHA384Length), @@ -798,6 +800,7 @@ namespace SabreTools.Library.DatFiles case ItemType.Disk: state += "\t\t\n" + "\t\t\t\t + /// Data RIPEMD160 hash + /// + public string RIPEMD160 + { + get { return _ripemd160.IsNullOrWhiteSpace() ? null : Utilities.ByteArrayToString(_ripemd160); } + set { _ripemd160 = Utilities.StringToByteArray(value); } + } + /// /// Data SHA-1 hash /// @@ -120,6 +130,7 @@ namespace SabreTools.Library.DatItems { this.Name = baseFile.Filename; _md5 = baseFile.MD5; + _ripemd160 = baseFile.RIPEMD160; _sha1 = baseFile.SHA1; _sha256 = baseFile.SHA256; _sha384 = baseFile.SHA384; @@ -172,6 +183,7 @@ namespace SabreTools.Library.DatItems Source = this.Source, _md5 = this._md5, + _ripemd160 = this._ripemd160, _sha1 = this._sha1, _sha256 = this._sha256, _sha384 = this._sha384, @@ -194,6 +206,7 @@ namespace SabreTools.Library.DatItems CRC = null, MD5 = this.MD5, + RIPEMD160 = this.RIPEMD160, SHA1 = this.SHA1, SHA256 = this.SHA256, SHA384 = this.SHA384, @@ -260,6 +273,7 @@ namespace SabreTools.Library.DatItems if ((this.ItemStatus == ItemStatus.Nodump && newOther.ItemStatus == ItemStatus.Nodump) && (this.Name == newOther.Name) && (this._md5.IsNullOrWhiteSpace() && newOther._md5.IsNullOrWhiteSpace()) + && (this._ripemd160.IsNullOrWhiteSpace() && newOther._ripemd160.IsNullOrWhiteSpace()) && (this._sha1.IsNullOrWhiteSpace() && newOther._sha1.IsNullOrWhiteSpace()) && (this._sha256.IsNullOrWhiteSpace() && newOther._sha256.IsNullOrWhiteSpace()) && (this._sha384.IsNullOrWhiteSpace() && newOther._sha384.IsNullOrWhiteSpace()) @@ -270,6 +284,7 @@ namespace SabreTools.Library.DatItems // If we can determine that the disks have no non-empty hashes in common, we return false else if ((this._md5.IsNullOrWhiteSpace() || newOther._md5.IsNullOrWhiteSpace()) + && (this._ripemd160.IsNullOrWhiteSpace() || newOther._ripemd160.IsNullOrWhiteSpace()) && (this._sha1.IsNullOrWhiteSpace() || newOther._sha1.IsNullOrWhiteSpace()) && (this._sha256.IsNullOrWhiteSpace() || newOther._sha256.IsNullOrWhiteSpace()) && (this._sha384.IsNullOrWhiteSpace() || newOther._sha384.IsNullOrWhiteSpace()) @@ -280,6 +295,7 @@ namespace SabreTools.Library.DatItems // Otherwise if we get a partial match else if (((this._md5.IsNullOrWhiteSpace() || newOther._md5.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._md5, newOther._md5)) + && ((this._ripemd160.IsNullOrWhiteSpace() || newOther._ripemd160.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._ripemd160, newOther._ripemd160)) && ((this._sha1.IsNullOrWhiteSpace() || newOther._sha1.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha1, newOther._sha1)) && ((this._sha256.IsNullOrWhiteSpace() || newOther._sha256.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha256, newOther._sha256)) && ((this._sha384.IsNullOrWhiteSpace() || newOther._sha384.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha384, newOther._sha384)) diff --git a/SabreTools.Library/DatItems/Rom.cs b/SabreTools.Library/DatItems/Rom.cs index 0738f69b..87169b7c 100644 --- a/SabreTools.Library/DatItems/Rom.cs +++ b/SabreTools.Library/DatItems/Rom.cs @@ -14,6 +14,7 @@ namespace SabreTools.Library.DatItems private byte[] _crc; // 8 bytes private byte[] _md5; // 16 bytes + private byte[] _ripemd160; // 20 bytes private byte[] _sha1; // 20 bytes private byte[] _sha256; // 32 bytes private byte[] _sha384; // 48 bytes @@ -51,6 +52,15 @@ namespace SabreTools.Library.DatItems set { _md5 = Utilities.StringToByteArray(value); } } + /// + /// File RIPEMD160 hash + /// + public string RIPEMD160 + { + get { return _ripemd160.IsNullOrWhiteSpace() ? null : Utilities.ByteArrayToString(_ripemd160); } + set { _ripemd160 = Utilities.StringToByteArray(value); } + } + /// /// File SHA-1 hash /// @@ -153,6 +163,10 @@ namespace SabreTools.Library.DatItems { _md5 = null; } + if ((omitFromScan & Hash.RIPEMD160) == 0) + { + _ripemd160 = null; + } if ((omitFromScan & Hash.SHA1) == 0) { _sha1 = null; @@ -188,6 +202,7 @@ namespace SabreTools.Library.DatItems this.Size = baseFile.Size ?? -1; _crc = baseFile.CRC; _md5 = baseFile.MD5; + _ripemd160 = baseFile.RIPEMD160; _sha1 = baseFile.SHA1; _sha256 = baseFile.SHA256; _sha384 = baseFile.SHA384; @@ -243,6 +258,7 @@ namespace SabreTools.Library.DatItems Size = this.Size, _crc = this._crc, _md5 = this._md5, + _ripemd160 = this._ripemd160, _sha1 = this._sha1, _sha256 = this._sha256, _sha384 = this._sha384, @@ -274,6 +290,7 @@ namespace SabreTools.Library.DatItems && (this.Name == newOther.Name) && (this._crc.IsNullOrWhiteSpace() && newOther._crc.IsNullOrWhiteSpace()) && (this._md5.IsNullOrWhiteSpace() && newOther._md5.IsNullOrWhiteSpace()) + && (this._ripemd160.IsNullOrWhiteSpace() && newOther._ripemd160.IsNullOrWhiteSpace()) && (this._sha1.IsNullOrWhiteSpace() && newOther._sha1.IsNullOrWhiteSpace()) && (this._sha256.IsNullOrWhiteSpace() && newOther._sha256.IsNullOrWhiteSpace()) && (this._sha384.IsNullOrWhiteSpace() && newOther._sha384.IsNullOrWhiteSpace()) @@ -285,6 +302,7 @@ namespace SabreTools.Library.DatItems // If we can determine that the roms have no non-empty hashes in common, we return false else if ((this._crc.IsNullOrWhiteSpace() || newOther._crc.IsNullOrWhiteSpace()) && (this._md5.IsNullOrWhiteSpace() || newOther._md5.IsNullOrWhiteSpace()) + && (this._ripemd160.IsNullOrWhiteSpace() || newOther._ripemd160.IsNullOrWhiteSpace()) && (this._sha1.IsNullOrWhiteSpace() || newOther._sha1.IsNullOrWhiteSpace()) && (this._sha256.IsNullOrWhiteSpace() || newOther._sha256.IsNullOrWhiteSpace()) && (this._sha384.IsNullOrWhiteSpace() || newOther._sha384.IsNullOrWhiteSpace()) @@ -297,6 +315,7 @@ namespace SabreTools.Library.DatItems else if ((this.Size == -1) && ((this._crc.IsNullOrWhiteSpace() || newOther._crc.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._crc, newOther._crc)) && ((this._md5.IsNullOrWhiteSpace() || newOther._md5.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._md5, newOther._md5)) + && ((this._ripemd160.IsNullOrWhiteSpace() || newOther._ripemd160.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._ripemd160, newOther._ripemd160)) && ((this._sha1.IsNullOrWhiteSpace() || newOther._sha1.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha1, newOther._sha1)) && ((this._sha256.IsNullOrWhiteSpace() || newOther._sha256.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha256, newOther._sha256)) && ((this._sha384.IsNullOrWhiteSpace() || newOther._sha384.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha384, newOther._sha384)) @@ -309,6 +328,7 @@ namespace SabreTools.Library.DatItems else if ((this.Size == newOther.Size) && ((this._crc.IsNullOrWhiteSpace() || newOther._crc.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._crc, newOther._crc)) && ((this._md5.IsNullOrWhiteSpace() || newOther._md5.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._md5, newOther._md5)) + && ((this._ripemd160.IsNullOrWhiteSpace() || newOther._ripemd160.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._ripemd160, newOther._ripemd160)) && ((this._sha1.IsNullOrWhiteSpace() || newOther._sha1.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha1, newOther._sha1)) && ((this._sha256.IsNullOrWhiteSpace() || newOther._sha256.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha256, newOther._sha256)) && ((this._sha384.IsNullOrWhiteSpace() || newOther._sha384.IsNullOrWhiteSpace()) || Enumerable.SequenceEqual(this._sha384, newOther._sha384)) diff --git a/SabreTools.Library/Data/Constants.cs b/SabreTools.Library/Data/Constants.cs index a75f131f..e7f04aba 100644 --- a/SabreTools.Library/Data/Constants.cs +++ b/SabreTools.Library/Data/Constants.cs @@ -26,58 +26,63 @@ namespace SabreTools.Library.Data public const long SizeZero = 0; public const string CRCZero = "00000000"; - public static readonly byte[] CRCZeroBytes = { 0x00, 0x00, 0x00, 0x00 }; + public static readonly byte[] CRCZeroBytes = { 0x00, 0x00, 0x00, 0x00 }; public const string MD5Zero = "d41d8cd98f00b204e9800998ecf8427e"; - public static readonly byte[] MD5ZeroBytes = { 0xda, 0x39, 0xa3, 0xee, - 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, - 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09 }; + public static readonly byte[] MD5ZeroBytes = { 0xd4, 0x1d, 0x8c, 0xd9, + 0x8f, 0x00, 0xb2, 0x04, + 0xe9, 0x80, 0x09, 0x98, + 0xec, 0xf8, 0x42, 0x7e }; + public const string RIPEMD160Zero = "9c1185a5c5e9fc54612808977ee8f548b2258d31"; + public static readonly byte[] RIPEMD160ZeroBytes = { 0x9c, 0x11, 0x85, 0xa5, + 0xc5, 0xe9, 0xfc, 0x54, + 0x61, 0x28, 0x08, 0x97, + 0x7e, 0xe8, 0xf5, 0x48, + 0xb2, 0x25, 0x8d, 0x31 }; public const string SHA1Zero = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; - public static readonly byte[] SHA1ZeroBytes = { 0xda, 0x39, 0xa3, 0xee, - 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, - 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09 }; + public static readonly byte[] SHA1ZeroBytes = { 0xda, 0x39, 0xa3, 0xee, + 0x5e, 0x6b, 0x4b, 0x0d, + 0x32, 0x55, 0xbf, 0xef, + 0x95, 0x60, 0x18, 0x90, + 0xaf, 0xd8, 0x07, 0x09 }; public const string SHA256Zero = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; - public static readonly byte[] SHA256ZeroBytes = { 0xba, 0x78, 0x16, 0xbf, - 0x8f, 0x01, 0xcf, 0xea, - 0x41, 0x41, 0x40, 0xde, - 0x5d, 0xae, 0x22, 0x23, - 0xb0, 0x03, 0x61, 0xa3, - 0x96, 0x17, 0x7a, 0x9c, - 0xb4, 0x10, 0xff, 0x61, - 0xf2, 0x00, 0x15, 0xad }; + public static readonly byte[] SHA256ZeroBytes = { 0xba, 0x78, 0x16, 0xbf, + 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, + 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, + 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad }; public const string SHA384Zero = "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7"; - public static readonly byte[] SHA384ZeroBytes = { 0xcb, 0x00, 0x75, 0x3f, - 0x45, 0xa3, 0x5e, 0x8b, - 0xb5, 0xa0, 0x3d, 0x69, - 0x9a, 0xc6, 0x50, 0x07, - 0x27, 0x2c, 0x32, 0xab, - 0x0e, 0xde, 0xd1, 0x63, - 0x1a, 0x8b, 0x60, 0x5a, - 0x43, 0xff, 0x5b, 0xed, - 0x80, 0x86, 0x07, 0x2b, - 0xa1, 0xe7, 0xcc, 0x23, - 0x58, 0xba, 0xec, 0xa1, - 0x34, 0xc8, 0x25, 0xa7 }; + public static readonly byte[] SHA384ZeroBytes = { 0xcb, 0x00, 0x75, 0x3f, + 0x45, 0xa3, 0x5e, 0x8b, + 0xb5, 0xa0, 0x3d, 0x69, + 0x9a, 0xc6, 0x50, 0x07, + 0x27, 0x2c, 0x32, 0xab, + 0x0e, 0xde, 0xd1, 0x63, + 0x1a, 0x8b, 0x60, 0x5a, + 0x43, 0xff, 0x5b, 0xed, + 0x80, 0x86, 0x07, 0x2b, + 0xa1, 0xe7, 0xcc, 0x23, + 0x58, 0xba, 0xec, 0xa1, + 0x34, 0xc8, 0x25, 0xa7 }; public const string SHA512Zero = "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"; - public static readonly byte[] SHA512ZeroBytes = { 0xdd, 0xaf, 0x35, 0xa1, - 0x93, 0x61, 0x7a, 0xba, - 0xcc, 0x41, 0x73, 0x49, - 0xae, 0x20, 0x41, 0x31, - 0x12, 0xe6, 0xfa, 0x4e, - 0x89, 0xa9, 0x7e, 0xa2, - 0x0a, 0x9e, 0xee, 0xe6, - 0x4b, 0x55, 0xd3, 0x9a, - 0x21, 0x92, 0x99, 0x2a, - 0x27, 0x4f, 0xc1, 0xa8, - 0x36, 0xba, 0x3c, 0x23, - 0xa3, 0xfe, 0xeb, 0xbd, - 0x45, 0x4d, 0x44, 0x23, - 0x64, 0x3c, 0xe8, 0x0e, - 0x2a, 0x9a, 0xc9, 0x4f, - 0xa5, 0x4c, 0xa4, 0x9f }; + public static readonly byte[] SHA512ZeroBytes = { 0xdd, 0xaf, 0x35, 0xa1, + 0x93, 0x61, 0x7a, 0xba, + 0xcc, 0x41, 0x73, 0x49, + 0xae, 0x20, 0x41, 0x31, + 0x12, 0xe6, 0xfa, 0x4e, + 0x89, 0xa9, 0x7e, 0xa2, + 0x0a, 0x9e, 0xee, 0xe6, + 0x4b, 0x55, 0xd3, 0x9a, + 0x21, 0x92, 0x99, 0x2a, + 0x27, 0x4f, 0xc1, 0xa8, + 0x36, 0xba, 0x3c, 0x23, + 0xa3, 0xfe, 0xeb, 0xbd, + 0x45, 0x4d, 0x44, 0x23, + 0x64, 0x3c, 0xe8, 0x0e, + 0x2a, 0x9a, 0xc9, 0x4f, + 0xa5, 0x4c, 0xa4, 0x9f }; #endregion @@ -233,6 +238,7 @@ namespace SabreTools.Library.Data + @@ -243,6 +249,7 @@ namespace SabreTools.Library.Data + @@ -290,6 +297,7 @@ namespace SabreTools.Library.Data + @@ -302,6 +310,7 @@ namespace SabreTools.Library.Data + @@ -468,6 +477,7 @@ namespace SabreTools.Library.Data + @@ -481,6 +491,7 @@ namespace SabreTools.Library.Data + @@ -503,6 +514,7 @@ namespace SabreTools.Library.Data public const int CRCLength = 8; public const int MD5Length = 32; + public const int RIPEMD160Length = 40; public const int SHA1Length = 40; public const int SHA256Length = 64; public const int SHA384Length = 96; diff --git a/SabreTools.Library/Data/Enums.cs b/SabreTools.Library/Data/Enums.cs index da489b8e..d71c2e42 100644 --- a/SabreTools.Library/Data/Enums.cs +++ b/SabreTools.Library/Data/Enums.cs @@ -241,6 +241,7 @@ Game, CRC, MD5, + RIPEMD160, SHA1, SHA256, SHA384, @@ -299,6 +300,7 @@ Size, CRC, MD5, + RIPEMD160, SHA1, SHA256, SHA384, @@ -363,6 +365,7 @@ // Disk MD5, + RIPEMD160, SHA1, SHA256, SHA384, diff --git a/SabreTools.Library/Data/Flags.cs b/SabreTools.Library/Data/Flags.cs index dba9adf5..bf8539e4 100644 --- a/SabreTools.Library/Data/Flags.cs +++ b/SabreTools.Library/Data/Flags.cs @@ -350,25 +350,30 @@ namespace SabreTools.Library.Data /// RedumpMD5 = 1 << 18, + /// + /// RIPEMD160 hash list + /// + RedumpRIPEMD160 = 1 << 19, + /// /// SHA-1 hash list /// - RedumpSHA1 = 1 << 19, + RedumpSHA1 = 1 << 20, /// /// SHA-256 hash list /// - RedumpSHA256 = 1 << 20, + RedumpSHA256 = 1 << 21, /// /// SHA-384 hash list /// - RedumpSHA384 = 1 << 21, + RedumpSHA384 = 1 << 22, /// /// SHA-512 hash list /// - RedumpSHA512 = 1 << 22, + RedumpSHA512 = 1 << 23, #endregion @@ -388,11 +393,12 @@ namespace SabreTools.Library.Data SHA256 = 1 << 3, SHA384 = 1 << 4, SHA512 = 1 << 5, + RIPEMD160 = 1 << 6, // Special combinations Standard = CRC | MD5 | SHA1, - DeepHashes = SHA256 | SHA384 | SHA512, - SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512, + DeepHashes = SHA256 | SHA384 | SHA512 | RIPEMD160, + SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | RIPEMD160, } /// diff --git a/SabreTools.Library/FileTypes/BaseFile.cs b/SabreTools.Library/FileTypes/BaseFile.cs index f3e01481..bcd83ba2 100644 --- a/SabreTools.Library/FileTypes/BaseFile.cs +++ b/SabreTools.Library/FileTypes/BaseFile.cs @@ -21,6 +21,7 @@ namespace SabreTools.Library.FileTypes public long? Size { get; set; } public byte[] CRC { get; set; } public byte[] MD5 { get; set; } + public byte[] RIPEMD160 { get; set; } public byte[] SHA1 { get; set; } public byte[] SHA256 { get; set; } public byte[] SHA384 { get; set; } @@ -49,13 +50,13 @@ namespace SabreTools.Library.FileTypes if (getHashes) { BaseFile temp = Utilities.GetFileInfo(this.Filename); - if (temp != null) { this.Parent = temp.Parent; this.Date = temp.Date; this.CRC = temp.CRC; this.MD5 = temp.MD5; + this.RIPEMD160 = temp.RIPEMD160; this.SHA1 = temp.SHA1; this.SHA256 = temp.SHA256; this.SHA384 = temp.SHA384; @@ -77,13 +78,13 @@ namespace SabreTools.Library.FileTypes if (getHashes) { BaseFile temp = Utilities.GetStreamInfo(stream, stream.Length); - if(temp != null) { this.Parent = temp.Parent; this.Date = temp.Date; this.CRC = temp.CRC; this.MD5 = temp.MD5; + this.RIPEMD160 = temp.RIPEMD160; this.SHA1 = temp.SHA1; this.SHA256 = temp.SHA256; this.SHA384 = temp.SHA384; @@ -93,31 +94,6 @@ namespace SabreTools.Library.FileTypes } - /// - /// Create a new BaseFile from the given metadata - /// - /// Name of the file to use - /// Parent folder or archive - /// File date - /// CRC hash as a byte array - /// MD5 hash as a byte array - /// SHA-1 hash as a byte array - /// SHA-256 hash as a byte array - /// SHA-384 hash as a byte array - /// SHA-512 hash as a byte array - public BaseFile(string filename, string parent, string date, byte[] crc, byte[] md5, byte[] sha1, byte[] sha256, byte[] sha384, byte[] sha512) - { - this.Filename = filename; - this.Parent = parent; - this.Date = date; - this.CRC = crc; - this.MD5 = md5; - this.SHA1 = sha1; - this.SHA256 = sha256; - this.SHA384 = sha384; - this.SHA512 = sha512; - } - #endregion } } diff --git a/SabreTools.Library/FileTypes/CHDFile.cs b/SabreTools.Library/FileTypes/CHDFile.cs index a08ec7e9..7f38515c 100644 --- a/SabreTools.Library/FileTypes/CHDFile.cs +++ b/SabreTools.Library/FileTypes/CHDFile.cs @@ -158,21 +158,15 @@ namespace SabreTools.Library.FileTypes m_br = new BinaryReader(chdstream); _headerVersion = ValidateHeaderVersion(); - if (_headerVersion != null) { byte[] hash = GetHashFromHeader(); - if (hash != null) { if (hash.Length == Constants.MD5Length) - { this.MD5 = hash; - } else if (hash.Length == Constants.SHA1Length) - { this.SHA1 = hash; - } } } } @@ -197,9 +191,7 @@ namespace SabreTools.Library.FileTypes // If no signature could be read, return null if (m_signature == null || m_signature.Length == 0) - { return null; - } if (!m_signature.StartsWith(Constants.CHDSignature, exact: true)) { diff --git a/SabreTools.Library/FileTypes/GZipArchive.cs b/SabreTools.Library/FileTypes/GZipArchive.cs index 732a4033..e8c94289 100644 --- a/SabreTools.Library/FileTypes/GZipArchive.cs +++ b/SabreTools.Library/FileTypes/GZipArchive.cs @@ -243,7 +243,7 @@ namespace SabreTools.Library.FileTypes var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); - BaseFile gzipEntryRom = Utilities.GetStreamInfo(gzstream, gzstream.Length, omitFromScan: omitFromScan); + BaseFile gzipEntryRom = Utilities.GetStreamInfo(gzstream, gzstream.Length, omitFromScan); gzipEntryRom.Filename = gz.Filename(0); gzipEntryRom.Parent = gamename; gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null); diff --git a/SabreTools.Library/FileTypes/RarArchive.cs b/SabreTools.Library/FileTypes/RarArchive.cs index d84debea..bdac57c7 100644 --- a/SabreTools.Library/FileTypes/RarArchive.cs +++ b/SabreTools.Library/FileTypes/RarArchive.cs @@ -215,7 +215,7 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile rarEntryRom = Utilities.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); + BaseFile rarEntryRom = Utilities.GetStreamInfo(entryStream, entry.Size, omitFromScan); rarEntryRom.Filename = entry.Key; rarEntryRom.Parent = gamename; rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/SevenZipArchive.cs b/SabreTools.Library/FileTypes/SevenZipArchive.cs index 1d0c17d7..5ff66cac 100644 --- a/SabreTools.Library/FileTypes/SevenZipArchive.cs +++ b/SabreTools.Library/FileTypes/SevenZipArchive.cs @@ -328,7 +328,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true); + BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.UncompressedSize(i), omitFromScan, true); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; found.Add(zipEntryRom); diff --git a/SabreTools.Library/FileTypes/TapeArchive.cs b/SabreTools.Library/FileTypes/TapeArchive.cs index e5e575ee..25a009cb 100644 --- a/SabreTools.Library/FileTypes/TapeArchive.cs +++ b/SabreTools.Library/FileTypes/TapeArchive.cs @@ -218,7 +218,7 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile tarEntryRom = Utilities.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); + BaseFile tarEntryRom = Utilities.GetStreamInfo(entryStream, entry.Size, omitFromScan); tarEntryRom.Filename = entry.Key; tarEntryRom.Parent = gamename; tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/ZipArchive.cs b/SabreTools.Library/FileTypes/ZipArchive.cs index 9754704e..e5e951e0 100644 --- a/SabreTools.Library/FileTypes/ZipArchive.cs +++ b/SabreTools.Library/FileTypes/ZipArchive.cs @@ -328,7 +328,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true); + BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.UncompressedSize(i), omitFromScan, true); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; string convertedDate = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/README.1ST b/SabreTools.Library/README.1ST index 4c476699..2defe6b1 100644 --- a/SabreTools.Library/README.1ST +++ b/SabreTools.Library/README.1ST @@ -176,6 +176,10 @@ Options: This allows the user to skip calculating the MD5 for each of the files which will speed up the creation of the DAT. + -nr160, --skip-ripemd160 Don't include RIPEMD160 in output + This allows the user to skip calculating the RIPEMD160 for each of + the files which will speed up the creation of the DAT. + -ns, --skip-sha1 Don't include SHA-1 in output This allows the user to skip calculating the SHA-1 for each of the files which will speed up the creation of the DAT. @@ -222,6 +226,7 @@ Options: msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter + ripemd160 - RIPEMD160 sd, sabredat - SabreDat XML sfv - SFV sha1 - SHA1 @@ -417,6 +422,18 @@ Options: the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed. + -ripemd160=, --ripemd160= Filter by RIPEMD160 hash + Include only items with this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + + -nripemd160=, --not-ripemd160= Filter by not RIPEMD160 hash + Include only items without this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + -sha1=, --sha1= Filter by SHA-1 hash Include only items with this SHA-1 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for @@ -751,6 +768,7 @@ Options: msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter + ripemd160 - RIPEMD160 sd, sabredat - SabreDat XML sfv - SFV sha1 - SHA1 @@ -924,6 +942,7 @@ Options: msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter + ripemd160 - RIPEMD160 sd, sabredat - SabreDat XML sfv - SFV sha1 - SHA1 @@ -949,6 +968,7 @@ Options: - %publisher% - Replaced with game Publisher - %crc% - Replaced with the CRC - %md5% - Replaced with the MD5 + - %ripemd160% - Replaced with the RIPEMD160 - %sha1% - Replaced with the SHA-1 - %sha256% - Replaced with the SHA-256 - %sha384% - Replaced with the SHA-384 @@ -1084,6 +1104,10 @@ Options: By default, all available hashes will be written out to the DAT. This will remove all MD5 hashes from the output file(s). + -rripemd160, --remove-ripemd160 Remove RIPEMD160 hashes from the output + By default, all available hashes will be written out to the DAT. This + will remove all RIPEMD160 hashes from the output file(s). + -rsha1, --remove-sha1 Remove SHA-1 hashes from the output By default, all available hashes will be written out to the DAT. This will remove all SHA-1 hashes from the output file(s). @@ -1371,6 +1395,18 @@ Options: the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed. + -ripemd160=, --ripemd160= Filter by RIPEMD160 hash + Include only items with this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + + -nripemd160=, --not-ripemd160= Filter by not RIPEMD160 hash + Include only items without this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + -sha1=, --sha1= Filter by SHA-1 hash Include only items with this SHA-1 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for @@ -1616,6 +1652,18 @@ Options: the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed. + -ripemd160=, --ripemd160= Filter by RIPEMD160 hash + Include only items with this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + + -nripemd160=, --not-ripemd160= Filter by not RIPEMD160 hash + Include only items without this RIPEMD160 hash in the output. + Additionally, the user can specify an exact match or full C#-style + regex for pattern matching. Multiple instances of this flag are + allowed. + -sha1=, --sha1= Filter by SHA-1 hash Include only items with this SHA-1 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for @@ -1827,6 +1875,7 @@ This section contains remappings from old flag names to new ones for the purpose -ool, --output-ol -> -ot=ol, --output-type=offlinelist -ool, --output-offlinelist -> -ot=ol, --output-type=offlinelist -or, --output-rc -> -ot=rc, --output-type=romcenter +-oripemd160, --output-ripemd160 -> -ot=ripemd160, --output-type=ripemd160 -or, --output-romcenter -> -ot=rc, --output-type=romcenter -os, --output-sd -> -ot=sd, --output-type=sabredat -os, --output-sabredat -> -ot=sd, --output-type=sabredat diff --git a/SabreTools.Library/Reports/Html.cs b/SabreTools.Library/Reports/Html.cs index 09cbc4c9..f9cca4e0 100644 --- a/SabreTools.Library/Reports/Html.cs +++ b/SabreTools.Library/Reports/Html.cs @@ -60,6 +60,7 @@ namespace SabreTools.Library.Reports + "" + _datFile.DiskCount + "" + "" + _datFile.CRCCount + "" + "" + _datFile.MD5Count + "" + + "" + _datFile.RIPEMD160Count + "" + "" + _datFile.SHA1Count + "" + "" + _datFile.SHA256Count + "" + (_baddumpCol ? "" + _datFile.BaddumpCount + "" : "") diff --git a/SabreTools.Library/Reports/SeparatedValue.cs b/SabreTools.Library/Reports/SeparatedValue.cs index ef1e74f7..f310cd4f 100644 --- a/SabreTools.Library/Reports/SeparatedValue.cs +++ b/SabreTools.Library/Reports/SeparatedValue.cs @@ -60,6 +60,7 @@ namespace SabreTools.Library.Reports + "\"" + _datFile.DiskCount + "\"{0}" + "\"" + _datFile.CRCCount + "\"{0}" + "\"" + _datFile.MD5Count + "\"{0}" + + "\"" + _datFile.RIPEMD160Count + "\"{0}" + "\"" + _datFile.SHA1Count + "\"{0}" + "\"" + _datFile.SHA256Count + "\"{0}" + "\"" + _datFile.SHA384Count + "\"{0}" diff --git a/SabreTools.Library/Reports/Textfile.cs b/SabreTools.Library/Reports/Textfile.cs index 3992ebb4..b51cf2ee 100644 --- a/SabreTools.Library/Reports/Textfile.cs +++ b/SabreTools.Library/Reports/Textfile.cs @@ -56,6 +56,7 @@ namespace SabreTools.Library.Reports Disks found: " + _datFile.DiskCount + @" Roms with CRC: " + _datFile.CRCCount + @" Roms with MD5: " + _datFile.MD5Count + @" + Roms with RIPEMD160: " + _datFile.RIPEMD160Count + @" Roms with SHA-1: " + _datFile.SHA1Count + @" Roms with SHA-256: " + _datFile.SHA256Count + @" Roms with SHA-384: " + _datFile.SHA384Count + @" diff --git a/SabreTools.Library/SabreTools.Library.csproj b/SabreTools.Library/SabreTools.Library.csproj index c4a9f734..03371c77 100644 --- a/SabreTools.Library/SabreTools.Library.csproj +++ b/SabreTools.Library/SabreTools.Library.csproj @@ -6,6 +6,10 @@ Debug;Release;Mono AnyCPU;x64 + + + NET_FRAMEWORK + DEBUG;TRACE;MONO diff --git a/SabreTools.Library/Tools/Hasher.cs b/SabreTools.Library/Tools/Hasher.cs new file mode 100644 index 00000000..9b307792 --- /dev/null +++ b/SabreTools.Library/Tools/Hasher.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; + +using SabreTools.Library.Data; + +namespace SabreTools.Library.Tools +{ + public class Hasher + { + public Hash HashType { get; private set; } + private IDisposable _hasher; + + public Hasher(Hash hashType) + { + this.HashType = hashType; + GetHasher(); + } + + /// + /// Generate the correct hashing class based on the hash type + /// + private void GetHasher() + { + switch (HashType) + { + case Hash.CRC: + _hasher = new OptimizedCRC(); + break; + + case Hash.MD5: + _hasher = MD5.Create(); + break; + +#if NET_FRAMEWORK + case Hash.RIPEMD160: + _hasher = RIPEMD160.Create(); + break; +#endif + + case Hash.SHA1: + _hasher = SHA1.Create(); + break; + + case Hash.SHA256: + _hasher = SHA256.Create(); + break; + + case Hash.SHA384: + _hasher = SHA384.Create(); + break; + + case Hash.SHA512: + _hasher = SHA512.Create(); + break; + } + } + + public void Dispose() + { + _hasher.Dispose(); + } + + public async Task Process(byte[] buffer, int size) + { + switch (HashType) + { + case Hash.CRC: + await Task.Run(() => (_hasher as OptimizedCRC).Update(buffer, 0, size)); + break; + + case Hash.MD5: +#if NET_FRAMEWORK + case Hash.RIPEMD160: +#endif + case Hash.RIPEMD160: + case Hash.SHA1: + case Hash.SHA256: + case Hash.SHA384: + case Hash.SHA512: + await Task.Run(() => (_hasher as HashAlgorithm).TransformBlock(buffer, 0, size, null, 0)); + break; + } + } + + public async Task Finalize() + { + byte[] emptyBuffer = new byte[0]; + switch (HashType) + { + case Hash.CRC: + await Task.Run(() => (_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0)); + break; + + case Hash.MD5: +#if NET_FRAMEWORK + case Hash.RIPEMD160: +#endif + case Hash.SHA1: + case Hash.SHA256: + case Hash.SHA384: + case Hash.SHA512: + await Task.Run(() => (_hasher as HashAlgorithm).TransformFinalBlock(emptyBuffer, 0, 0)); + break; + } + } + + public byte[] GetHash() + { + switch (HashType) + { + case Hash.CRC: + return BitConverter.GetBytes((_hasher as OptimizedCRC).Value).Reverse().ToArray(); + + case Hash.MD5: +#if NET_FRAMEWORK + case Hash.RIPEMD160: +#endif + case Hash.SHA1: + case Hash.SHA256: + case Hash.SHA384: + case Hash.SHA512: + return (_hasher as HashAlgorithm).Hash; + } + + return null; + } + } +} diff --git a/SabreTools.Library/Tools/Utilities.cs b/SabreTools.Library/Tools/Utilities.cs index f13cae89..f8c364c6 100644 --- a/SabreTools.Library/Tools/Utilities.cs +++ b/SabreTools.Library/Tools/Utilities.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; +using System.Threading.Tasks; using System.Xml; using System.Xml.Schema; @@ -15,6 +16,7 @@ using SabreTools.Library.DatItems; using SabreTools.Library.FileTypes; using SabreTools.Library.Reports; using SabreTools.Library.Skippers; +using Compress.ThreadReaders; #if MONO using System.IO; @@ -638,6 +640,8 @@ namespace SabreTools.Library.Tools return new OpenMSX(baseDat); case DatFormat.RedumpMD5: return new Hashfile(baseDat, Hash.MD5); + case DatFormat.RedumpRIPEMD160: + return new Hashfile(baseDat, Hash.RIPEMD160); case DatFormat.RedumpSFV: return new Hashfile(baseDat, Hash.CRC); case DatFormat.RedumpSHA1: @@ -759,6 +763,8 @@ namespace SabreTools.Library.Tools case "rc": case "romcenter": return DatFormat.RomCenter; + case "ripemd160": + return DatFormat.RedumpRIPEMD160; case "sd": case "sabredat": return DatFormat.SabreDat; @@ -857,6 +863,8 @@ namespace SabreTools.Library.Tools return Field.RebuildTo; case "region": return Field.Region; + case "ripemd160": + return Field.RIPEMD160; case "romof": return Field.RomOf; case "runnable": @@ -1242,6 +1250,8 @@ namespace SabreTools.Library.Tools return DatFormat.CSV; case "md5": return DatFormat.RedumpMD5; + case "ripemd160": + return DatFormat.RedumpRIPEMD160; case "sfv": return DatFormat.RedumpSFV; case "sha1": @@ -1366,13 +1376,11 @@ namespace SabreTools.Library.Tools /// /// Filename to get information from /// Hash flag saying what hashes should not be calculated (defaults to none) - /// Set a >0 number for getting hash for part of the file, 0 otherwise (default) /// True if the file Date should be included, false otherwise (default) /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// True if CHDs should be treated like regular files, false otherwise /// Populated BaseFile object if success, empty one on error - public static BaseFile GetFileInfo(string input, Hash omitFromScan = 0x0, - long offset = 0, bool date = false, string header = null, bool chdsAsFiles = true) + public static BaseFile GetFileInfo(string input, Hash omitFromScan = 0x0, bool date = false, string header = null, bool chdsAsFiles = true) { // Add safeguard if file doesn't exist if (!File.Exists(input)) @@ -1395,7 +1403,7 @@ namespace SabreTools.Library.Tools // Transform the stream and get the information from it rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true); - baseFile = GetStreamInfo(outputStream, outputStream.Length, omitFromScan: omitFromScan, keepReadOpen: false, chdsAsFiles: chdsAsFiles); + baseFile = GetStreamInfo(outputStream, outputStream.Length, omitFromScan, false, chdsAsFiles); // Dispose of the streams outputStream.Dispose(); @@ -1405,13 +1413,13 @@ namespace SabreTools.Library.Tools else { long length = new FileInfo(input).Length; - baseFile = GetStreamInfo(TryOpenRead(input), length, omitFromScan, offset, keepReadOpen: false, chdsAsFiles: chdsAsFiles); + baseFile = GetStreamInfo(TryOpenRead(input), length, omitFromScan, false, chdsAsFiles); } } else { long length = new FileInfo(input).Length; - baseFile = GetStreamInfo(TryOpenRead(input), length, omitFromScan, offset, keepReadOpen: false, chdsAsFiles: chdsAsFiles); + baseFile = GetStreamInfo(TryOpenRead(input), length, omitFromScan, false, chdsAsFiles); } // Add unique data from the file @@ -2097,172 +2105,130 @@ namespace SabreTools.Library.Tools /// Filename to get information from /// Size of the input stream /// Hash flag saying what hashes should not be calculated (defaults to none) - /// Set a >0 number for getting hash for part of the file, 0 otherwise (default) /// True if the underlying read stream should be kept open, false otherwise /// True if CHDs should be treated like regular files, false otherwise /// Populated BaseFile object if success, empty one on error - public static BaseFile GetStreamInfo(Stream input, long size, Hash omitFromScan = 0x0, - long? offset = null, bool keepReadOpen = false, bool chdsAsFiles = true) + public static BaseFile GetStreamInfo(Stream input, long size, Hash omitFromScan = 0x0, bool keepReadOpen = false, bool chdsAsFiles = true) { - // We first check to see if it's a CHD - if (chdsAsFiles == false && GetCHDInfo(input) != null) + return GetStreamInfoAsync(input, size, omitFromScan, keepReadOpen, chdsAsFiles).ConfigureAwait(false).GetAwaiter().GetResult(); + } + + /// + /// Retrieve file information for a single file + /// + /// Filename to get information from + /// Size of the input stream + /// Hash flag saying what hashes should not be calculated (defaults to none) + /// True if the underlying read stream should be kept open, false otherwise + /// True if CHDs should be treated like regular files, false otherwise + /// Populated BaseFile object if success, empty one on error + public static async Task GetStreamInfoAsync(Stream input, long size, Hash omitFromScan = 0x0, bool keepReadOpen = false, bool chdsAsFiles = true) + { + // We first check to see if it's a CHD if we have to + if (!chdsAsFiles) { - // Seek to the starting position, if one is set - if (input.CanSeek && offset.HasValue) - { - try - { - if (offset < 0) - { - input.Seek(offset.Value, SeekOrigin.End); - } - else if (offset >= 0) - { - input.Seek(offset.Value, SeekOrigin.Begin); - } - } - catch (NotSupportedException) - { - Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); - } - catch (NotImplementedException) - { - Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed"); - } - } - - // Get the BaseFile from the information BaseFile chd = GetCHDInfo(input); + SeekIfPossible(input); - // Seek to the beginning of the stream if possible - try + // If we found a valid CHD + if (chd != null) { - input.Seek(0, SeekOrigin.Begin); - } - catch (NotSupportedException) - { - Globals.Logger.Verbose("Stream does not support seeking to beginning. Stream position not changed"); - } - catch (NotImplementedException) - { - Globals.Logger.Verbose("Stream does not support seeking to beginning. Stream position not changed"); - } + if (!keepReadOpen) + input.Dispose(); - if (!keepReadOpen) - { - input.Dispose(); + return chd; } - - return chd; } - BaseFile rom = new BaseFile() + BaseFile rom = new BaseFile() { Size = size, }; try { - // Initialize the hashers - OptimizedCRC crc = new OptimizedCRC(); - MD5 md5 = MD5.Create(); - SHA1 sha1 = SHA1.Create(); - SHA256 sha256 = SHA256.Create(); - SHA384 sha384 = SHA384.Create(); - SHA512 sha512 = SHA512.Create(); - - // Seek to the starting position, if one is set - if (input.CanSeek && offset.HasValue) + // Get a list of hashers to run over the buffer + List hashers = new List() { - try - { - if (offset < 0) - { - input.Seek(offset.Value, SeekOrigin.End); - } - else if (offset >= 0) - { - input.Seek(offset.Value, SeekOrigin.Begin); - } - } - catch (NotSupportedException) - { - Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); - } - catch (NotImplementedException) - { - Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); - } + new Hasher(Hash.CRC), + new Hasher(Hash.MD5), +#if NET_FRAMEWORK + new Hasher(Hash.RIPEMD160), +#endif + new Hasher(Hash.SHA1), + new Hasher(Hash.SHA256), + new Hasher(Hash.SHA384), + new Hasher(Hash.SHA512), + }; + + // Initialize the hashing helpers + var loadBuffer = new ThreadLoadBuffer(input); + int buffersize = 3 * 1024 * 1024; + byte[] buffer0 = new byte[buffersize]; + byte[] buffer1 = new byte[buffersize]; + + /* + Please note that some of the following code is adapted from + RomVault. This is a modified version of how RomVault does + threaded hashing. As such, some of the terminology and code + is the same, though variable names and comments may have + been tweaked to better fit this code base. + */ + + // Pre load the first buffer + int next = size > buffersize ? buffersize : (int)size; + input.Read(buffer0, 0, next); + int current = next; + size -= next; + bool bufferSelect = true; + + while (current > 0) + { + // Trigger the buffer load on the second buffer + next = size > buffersize ? buffersize : (int)size; + if (next > 0) + loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); + + byte[] buffer = bufferSelect ? buffer0 : buffer1; + + // Run hashes in parallel + await Task.WhenAll(hashers.Select(h => h.Process(buffer, current))); + + // Wait for the load buffer worker, if needed + if (next > 0) + loadBuffer.Wait(); + + // Setup for the next hashing step + current = next; + size -= next; + bufferSelect = !bufferSelect; } - byte[] buffer = new byte[32 * 1024 * 1024]; - int read; - while ((read = input.Read(buffer, 0, (int)Math.Min(size, buffer.Length))) > 0) - { - size -= read; - - crc.Update(buffer, 0, read); - if ((omitFromScan & Hash.MD5) == 0) - { - md5.TransformBlock(buffer, 0, read, buffer, 0); - } - if ((omitFromScan & Hash.SHA1) == 0) - { - sha1.TransformBlock(buffer, 0, read, buffer, 0); - } - if ((omitFromScan & Hash.SHA256) == 0) - { - sha256.TransformBlock(buffer, 0, read, buffer, 0); - } - if ((omitFromScan & Hash.SHA384) == 0) - { - sha384.TransformBlock(buffer, 0, read, buffer, 0); - } - if ((omitFromScan & Hash.SHA512) == 0) - { - sha512.TransformBlock(buffer, 0, read, buffer, 0); - } - - if (size <= 0) - break; - } - - crc.Update(buffer, 0, 0); - rom.CRC = BitConverter.GetBytes(crc.Value).Reverse().ToArray(); + // Finalize all hashing helpers + loadBuffer.Finish(); + await Task.WhenAll(hashers.Select(h => h.Finalize())); + // Get the results + if ((omitFromScan & Hash.CRC) == 0) + rom.CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash(); if ((omitFromScan & Hash.MD5) == 0) - { - md5.TransformFinalBlock(buffer, 0, 0); - rom.MD5 = md5.Hash; - } + rom.MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash(); +#if NET_FRAMEWORK + if ((omitFromScan & Hash.RIPEMD160) == 0) + rom.RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash(); +#endif if ((omitFromScan & Hash.SHA1) == 0) - { - sha1.TransformFinalBlock(buffer, 0, 0); - rom.SHA1 = sha1.Hash; - } + rom.SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash(); if ((omitFromScan & Hash.SHA256) == 0) - { - sha256.TransformFinalBlock(buffer, 0, 0); - rom.SHA256 = sha256.Hash; - } + rom.SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash(); if ((omitFromScan & Hash.SHA384) == 0) - { - sha384.TransformFinalBlock(buffer, 0, 0); - rom.SHA384 = sha384.Hash; - } + rom.SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash(); if ((omitFromScan & Hash.SHA512) == 0) - { - sha512.TransformFinalBlock(buffer, 0, 0); - rom.SHA512 = sha512.Hash; - } + rom.SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash(); // Dispose of the hashers - crc.Dispose(); - md5.Dispose(); - sha1.Dispose(); - sha256.Dispose(); - sha384.Dispose(); - sha512.Dispose(); + loadBuffer.Dispose(); + hashers.ForEach(h => h.Dispose()); } catch (IOException) { @@ -2270,27 +2236,10 @@ namespace SabreTools.Library.Tools } finally { - // Seek to the beginning of the stream if possible - if (input.CanSeek) - { - try - { - input.Seek(0, SeekOrigin.Begin); - } - catch (NotSupportedException) - { - Globals.Logger.Verbose("Stream does not support seeking to beginning. Stream position not changed"); - } - catch (NotImplementedException) - { - Globals.Logger.Verbose("Stream does not support seeking to beginning. Stream position not changed"); - } - } - if (!keepReadOpen) - { input.Dispose(); - } + else + SeekIfPossible(input); } return rom; @@ -2353,6 +2302,33 @@ namespace SabreTools.Library.Tools } } + /// + /// Seek to a specific point in the stream, if possible + /// + /// Input stream to try seeking on + /// Optional offset to seek to + public static void SeekIfPossible(Stream input, long offset = 0) + { + if (input.CanSeek) + { + try + { + if (offset < 0) + input.Seek(offset, SeekOrigin.End); + else if (offset >= 0) + input.Seek(offset, SeekOrigin.Begin); + } + catch (NotSupportedException) + { + Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); + } + catch (NotImplementedException) + { + Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed"); + } + } + } + #endregion #region String Manipulation @@ -2638,6 +2614,13 @@ namespace SabreTools.Library.Tools ? ((Disk)item).MD5 : Constants.MD5Zero)); break; + case SortedBy.RIPEMD160: + key = (item.ItemType == ItemType.Rom + ? ((Rom)item).RIPEMD160 + : (item.ItemType == ItemType.Disk + ? ((Disk)item).RIPEMD160 + : Constants.RIPEMD160Zero)); + break; case SortedBy.SHA1: key = (item.ItemType == ItemType.Rom ? ((Rom)item).SHA1 @@ -2874,6 +2857,7 @@ namespace SabreTools.Library.Tools case "csv": case "dat": case "md5": + case "ripemd160": case "sfv": case "sha1": case "sha256": diff --git a/SabreTools/SabreTools.Help.cs b/SabreTools/SabreTools.Help.cs index e273f2d2..64ad72b4 100644 --- a/SabreTools/SabreTools.Help.cs +++ b/SabreTools/SabreTools.Help.cs @@ -597,6 +597,18 @@ namespace SabreTools longDescription: "By default, all available hashes will be written out to the DAT. This will remove all MD5 hashes from the output file(s)."); } } // TODO: Remove + private static Feature _removeRipeMd160Flag + { + get + { + return new Feature( + "remove-ripemd160", + new List() { "-rripemd160", "--remove-ripemd160" }, + "Remove RIPEMD160 hashes from the output", + FeatureType.Flag, + longDescription: "By default, all available hashes will be written out to the DAT. This will remove all MD5 hashes from the output file(s)."); + } + } // TODO: Remove private static Feature _removeSha1Flag { get @@ -813,6 +825,18 @@ namespace SabreTools longDescription: "This allows the user to skip calculating the MD5 for each of the files which will speed up the creation of the DAT."); } } + private static Feature _skipRipeMd160Flag + { + get + { + return new Feature( + "skip-ripemd160", + new List() { "-nr160", "--skip-ripemd160" }, + "Include RIPEMD160 in output", // TODO: Invert this later + FeatureType.Flag, + longDescription: "This allows the user to skip calculating the RIPEMD160 for each of the files which will speed up the creation of the DAT."); + } + } private static Feature _skipSha1Flag { get @@ -1442,6 +1466,18 @@ Possible values are: None, Bios, Device, Mechanical"); longDescription: "Include only items without this MD5 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed."); } } + private static Feature _notRipeMd160ListInput + { + get + { + return new Feature( + "not-ripemd160", + new List() { "-nripemd160", "--not-ripemd160" }, + "Filter by not RIPEMD160 hash", + FeatureType.List, + longDescription: "Include only items without this RIPEMD160 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed."); + } + } private static Feature _notSha1ListInput { get @@ -1527,6 +1563,7 @@ Possible values are: msx, openmsx - openMSX Software List ol, offlinelist - OfflineList XML rc, romcenter - RomCenter + ripemd160 - RIPEMD160 sd, sabredat - SabreDat XML sfv - SFV sha1 - SHA1 @@ -1560,6 +1597,18 @@ Possible values are: tsv - Standardized Tab-Separated Value"); } } + private static Feature _ripeMd160ListInput + { + get + { + return new Feature( + "ripemd160", + new List() { "-ripemd160", "--ripemd160" }, + "Filter by RIPEMD160 hash", + FeatureType.List, + longDescription: "Include only items with this RIPEMD160 hash in the output. Additionally, the user can specify an exact match or full C#-style regex for pattern matching. Multiple instances of this flag are allowed."); + } + } private static Feature _sha1ListInput { get @@ -1876,6 +1925,7 @@ Some special strings that can be used: - %publisher% - Replaced with game Publisher - %crc% - Replaced with the CRC - %md5% - Replaced with the MD5 +- %ripemd160% - Replaced with the RIPEMD160 - %sha1% - Replaced with the SHA-1 - %sha256% - Replaced with the SHA-256 - %sha384% - Replaced with the SHA-384 @@ -2038,6 +2088,7 @@ Some special strings that can be used: FeatureType.Flag, longDescription: "Create a DAT file from an input directory or set of files. By default, this will output a DAT named based on the input directory and the current date. It will also treat all archives as possible games and add all three hashes (CRC, MD5, SHA-1) for each file."); datFromDir.AddFeature(_skipMd5Flag); + datFromDir.AddFeature(_skipRipeMd160Flag); datFromDir.AddFeature(_skipSha1Flag); datFromDir.AddFeature(_skipSha256Flag); datFromDir.AddFeature(_skipSha384Flag); @@ -2086,6 +2137,8 @@ Some special strings that can be used: datFromDir.AddFeature(_notCrcListInput); datFromDir.AddFeature(_md5ListInput); datFromDir.AddFeature(_notMd5ListInput); + datFromDir.AddFeature(_ripeMd160ListInput); + datFromDir.AddFeature(_notRipeMd160ListInput); datFromDir.AddFeature(_sha1ListInput); datFromDir.AddFeature(_notSha1ListInput); datFromDir.AddFeature(_sha256ListInput); @@ -2294,6 +2347,7 @@ The stats that are outputted are as follows: update.AddFeature(_cleanFlag); update.AddFeature(_removeUnicodeFlag); update.AddFeature(_removeMd5Flag); + update.AddFeature(_removeRipeMd160Flag); update.AddFeature(_removeSha1Flag); update.AddFeature(_removeSha256Flag); update.AddFeature(_removeSha384Flag); @@ -2361,6 +2415,8 @@ The stats that are outputted are as follows: update.AddFeature(_notCrcListInput); update.AddFeature(_md5ListInput); update.AddFeature(_notMd5ListInput); + update.AddFeature(_ripeMd160ListInput); + update.AddFeature(_notRipeMd160ListInput); update.AddFeature(_sha1ListInput); update.AddFeature(_notSha1ListInput); update.AddFeature(_sha256ListInput); @@ -2418,6 +2474,8 @@ The stats that are outputted are as follows: verify.AddFeature(_notCrcListInput); verify.AddFeature(_md5ListInput); verify.AddFeature(_notMd5ListInput); + verify.AddFeature(_ripeMd160ListInput); + verify.AddFeature(_notRipeMd160ListInput); verify.AddFeature(_sha1ListInput); verify.AddFeature(_notSha1ListInput); verify.AddFeature(_sha256ListInput); diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index 48deeb70..5445b16f 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -357,6 +357,10 @@ namespace SabreTools Globals.Logger.User("This flag '{0}' is depreciated, please use {1} instead", feat.Key, String.Join(", ", _excludeFieldListInput.Flags)); datHeader.ExcludeFields[(int)Field.MD5] = true; break; + case "remove-ripemd160": + Globals.Logger.User("This flag '{0}' is depreciated, please use {1} instead", feat.Key, String.Join(", ", _excludeFieldListInput.Flags)); + datHeader.ExcludeFields[(int)Field.RIPEMD160] = true; + break; case "remove-sha1": Globals.Logger.User("This flag '{0}' is depreciated, please use {1} instead", feat.Key, String.Join(", ", _excludeFieldListInput.Flags)); datHeader.ExcludeFields[(int)Field.SHA1] = true; @@ -415,6 +419,9 @@ namespace SabreTools case "skip-md5": omitFromScan |= Hash.MD5; break; + case "skip-ripemd160": + omitFromScan &= ~Hash.RIPEMD160; // This needs to be inverted later + break; case "skip-sha1": omitFromScan |= Hash.SHA1; break; @@ -596,6 +603,9 @@ namespace SabreTools case "not-md5": filter.MD5.NegativeSet.AddRange((List)feat.Value.GetValue()); break; + case "not-ripemd160": + filter.RIPEMD160.NegativeSet.AddRange((List)feat.Value.GetValue()); + break; case "not-sha1": filter.SHA1.NegativeSet.AddRange((List)feat.Value.GetValue()); break; @@ -631,6 +641,9 @@ namespace SabreTools statDatFormat |= Utilities.GetStatFormat(rt); } break; + case "ripemd160": + filter.RIPEMD160.PositiveSet.AddRange((List)feat.Value.GetValue()); + break; case "sha1": filter.SHA1.PositiveSet.AddRange((List)feat.Value.GetValue()); break;