From 15e30d7a759d3fd0e94d0064b9f42fe32b9b7305 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Thu, 24 Oct 2024 05:33:28 -0400 Subject: [PATCH] Reduce array allocations --- RombaSharp/Features/Archive.cs | 5 +++-- RombaSharp/Features/Dir2Dat.cs | 3 ++- RombaSharp/Features/RefreshDats.cs | 3 ++- RombaSharp/Features/RescanDepots.cs | 14 ++++++++------ SabreTools/Features/Batch.cs | 4 +++- SabreTools/Features/Verify.cs | 5 +++-- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/RombaSharp/Features/Archive.cs b/RombaSharp/Features/Archive.cs index e95e65dc..9bbc6173 100644 --- a/RombaSharp/Features/Archive.cs +++ b/RombaSharp/Features/Archive.cs @@ -58,6 +58,7 @@ have a current entry in the DAT index."; // Get feature flags bool noDb = GetBoolean(features, NoDbValue); bool onlyNeeded = GetBoolean(features, OnlyNeededValue); + HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; // First we want to get just all directories from the inputs List onlyDirs = []; @@ -71,8 +72,8 @@ have a current entry in the DAT index."; DatFile df = DatFile.Create(); foreach (string dir in onlyDirs) { - DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); - DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: hashes); + DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: hashes); } // Create an empty Dat for files that need to be rebuilt diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index b5d2c9d7..e179f15c 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -41,6 +41,7 @@ namespace RombaSharp.Features string? description = GetString(features, DescriptionStringValue); string? source = GetString(features, SourceStringValue); string? outdat = GetString(features, OutStringValue); + HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; // Ensure the output directory outdat = outdat.Ensure(create: true); @@ -56,7 +57,7 @@ namespace RombaSharp.Features DatFile datfile = DatFile.Create(); datfile.Header.SetFieldValue(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name); datfile.Header.SetFieldValue(SabreTools.Models.Metadata.Header.DescriptionKey, description); - DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: hashes); Writer.Write(datfile, outdat!); return true; } diff --git a/RombaSharp/Features/RefreshDats.cs b/RombaSharp/Features/RefreshDats.cs index 02ac8dfc..8b9f060e 100644 --- a/RombaSharp/Features/RefreshDats.cs +++ b/RombaSharp/Features/RefreshDats.cs @@ -44,6 +44,7 @@ contents of any changed dats."; // Get feature flags int workers = GetInt32(features, WorkersInt32Value); string? missingSha1s = GetString(features, MissingSha1sStringValue); + HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; // Make sure the db is set if (string.IsNullOrWhiteSpace(_db)) @@ -69,7 +70,7 @@ contents of any changed dats."; // First get a list of SHA-1's from the input DATs DatFile datroot = DatFile.Create(); datroot.Header.SetFieldValue(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT"); - DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: hashes); datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // Create a List of dat hashes in the database (SHA-1) diff --git a/RombaSharp/Features/RescanDepots.cs b/RombaSharp/Features/RescanDepots.cs index 00799fa9..9cd242fe 100644 --- a/RombaSharp/Features/RescanDepots.cs +++ b/RombaSharp/Features/RescanDepots.cs @@ -35,6 +35,8 @@ namespace RombaSharp.Features if (!base.ProcessFeatures(features)) return false; + HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; + logger.Error("This feature is not yet implemented: rescan-depots"); foreach (string depotname in Inputs) @@ -58,7 +60,7 @@ namespace RombaSharp.Features dbc.Open(); // If we have it, then check for all hashes that are in that depot - List hashes = []; + List sha1Hashes = []; string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); @@ -66,13 +68,13 @@ namespace RombaSharp.Features { while (sldr.Read()) { - hashes.Add(sldr.GetString(0)); + sha1Hashes.Add(sldr.GetString(0)); } } // Now rescan the depot itself DatFile depot = DatFile.Create(); - DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: hashes); depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // Set the base queries to use @@ -93,10 +95,10 @@ namespace RombaSharp.Features foreach (Rom rom in roms) { - if (hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!)) + if (sha1Hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!)) { dupehashes.Add(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!); - hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!); + sha1Hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!); } else if (!dupehashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!)) { @@ -162,7 +164,7 @@ JOIN crc JOIN md5 ON md5sha1.md5=md5.md5 WHERE sha1.sha1 IN "; - query += $"({string.Join("\",\"", hashes)}\")"; + query += $"({string.Join("\",\"", sha1Hashes)}\")"; slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index 2a0613da..6524d65a 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -278,10 +278,12 @@ Reset the internal state: reset();"; /// TODO: Should any of the other options be added for D2D? public override void Process(BatchState batchState) { + HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; + // Assume there could be multiple foreach (string input in Arguments) { - DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: hashes); } // TODO: We might not want to remove dates in the future diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 1362c8d9..bc77d1e5 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -53,6 +53,7 @@ namespace SabreTools.Features TreatAsFile asFiles = GetTreatAsFiles(features); bool hashOnly = GetBoolean(features, HashOnlyValue); bool quickScan = GetBoolean(features, QuickValue); + HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]; // If we are in individual mode, process each DAT on their own if (GetBoolean(features, IndividualValue)) @@ -91,7 +92,7 @@ namespace SabreTools.Features logger.User("Processing files:\n"); foreach (string input in Inputs) { - DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes); } Verification.VerifyGeneric(datdata, hashOnly); @@ -145,7 +146,7 @@ namespace SabreTools.Features logger.User("Processing files:\n"); foreach (string input in Inputs) { - DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]); + DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes); } Verification.VerifyGeneric(datdata, hashOnly);