claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce array allocations

Browse Source
This commit is contained in:
Matt Nadareski
2024-10-24 05:33:28 -04:00
parent e992f0d013
commit 15e30d7a75
6 changed files with 21 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
RombaSharp/Features/Archive.cs
View File

@@ -58,6 +58,7 @@ have a current entry in the DAT index.";
// Get feature flags // Get feature flags
bool noDb = GetBoolean(features, NoDbValue); bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue); bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// First we want to get just all directories from the inputs // First we want to get just all directories from the inputs
List<string> onlyDirs = []; List<string> onlyDirs = [];
@@ -71,8 +72,8 @@ have a current entry in the DAT index.";
DatFile df = DatFile.Create(); DatFile df = DatFile.Create();
foreach (string dir in onlyDirs) foreach (string dir in onlyDirs)
{ {
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: hashes);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: hashes);
} }
// Create an empty Dat for files that need to be rebuilt // Create an empty Dat for files that need to be rebuilt

3
RombaSharp/Features/Dir2Dat.cs
View File

@@ -41,6 +41,7 @@ namespace RombaSharp.Features
string? description = GetString(features, DescriptionStringValue); string? description = GetString(features, DescriptionStringValue);
string? source = GetString(features, SourceStringValue); string? source = GetString(features, SourceStringValue);
string? outdat = GetString(features, OutStringValue); string? outdat = GetString(features, OutStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Ensure the output directory // Ensure the output directory
outdat = outdat.Ensure(create: true); outdat = outdat.Ensure(create: true);
@@ -56,7 +57,7 @@ namespace RombaSharp.Features
DatFile datfile = DatFile.Create(); DatFile datfile = DatFile.Create();
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name); datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name);
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description); datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description);
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: hashes);
Writer.Write(datfile, outdat!); Writer.Write(datfile, outdat!);
return true; return true;
} }

3
RombaSharp/Features/RefreshDats.cs
View File

@@ -44,6 +44,7 @@ contents of any changed dats.";
// Get feature flags // Get feature flags
int workers = GetInt32(features, WorkersInt32Value); int workers = GetInt32(features, WorkersInt32Value);
string? missingSha1s = GetString(features, MissingSha1sStringValue); string? missingSha1s = GetString(features, MissingSha1sStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Make sure the db is set // Make sure the db is set
if (string.IsNullOrWhiteSpace(_db)) if (string.IsNullOrWhiteSpace(_db))
@@ -69,7 +70,7 @@ contents of any changed dats.";
// First get a list of SHA-1's from the input DATs // First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create(); DatFile datroot = DatFile.Create();
datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT"); datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT");
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: hashes);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1) // Create a List of dat hashes in the database (SHA-1)

14
RombaSharp/Features/RescanDepots.cs
View File

@@ -35,6 +35,8 @@ namespace RombaSharp.Features
if (!base.ProcessFeatures(features)) if (!base.ProcessFeatures(features))
return false; return false;
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
logger.Error("This feature is not yet implemented: rescan-depots"); logger.Error("This feature is not yet implemented: rescan-depots");
foreach (string depotname in Inputs) foreach (string depotname in Inputs)
@@ -58,7 +60,7 @@ namespace RombaSharp.Features
dbc.Open(); dbc.Open();
// If we have it, then check for all hashes that are in that depot // If we have it, then check for all hashes that are in that depot
List<string> hashes = []; List<string> sha1Hashes = [];
string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\""; string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
SqliteCommand slc = new SqliteCommand(query, dbc); SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader(); SqliteDataReader sldr = slc.ExecuteReader();
@@ -66,13 +68,13 @@ namespace RombaSharp.Features
{ {
while (sldr.Read()) while (sldr.Read())
{ {
hashes.Add(sldr.GetString(0)); sha1Hashes.Add(sldr.GetString(0));
} }
} }
// Now rescan the depot itself // Now rescan the depot itself
DatFile depot = DatFile.Create(); DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: hashes);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use // Set the base queries to use
@@ -93,10 +95,10 @@ namespace RombaSharp.Features
foreach (Rom rom in roms) foreach (Rom rom in roms)
{ {
if (hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!)) if (sha1Hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!))
{ {
dupehashes.Add(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!); dupehashes.Add(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!);
hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!); sha1Hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!);
} }
else if (!dupehashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!)) else if (!dupehashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!))
{ {
@@ -162,7 +164,7 @@ JOIN crc
JOIN md5 JOIN md5
ON md5sha1.md5=md5.md5 ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN "; WHERE sha1.sha1 IN ";
query += $"({string.Join("\",\"", hashes)}\")"; query += $"({string.Join("\",\"", sha1Hashes)}\")";
slc = new SqliteCommand(query, dbc); slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery(); slc.ExecuteNonQuery();

4
SabreTools/Features/Batch.cs
View File

@@ -278,10 +278,12 @@ Reset the internal state: reset();";
/// <remarks>TODO: Should any of the other options be added for D2D?</remarks> /// <remarks>TODO: Should any of the other options be added for D2D?</remarks>
public override void Process(BatchState batchState) public override void Process(BatchState batchState)
{ {
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Assume there could be multiple // Assume there could be multiple
foreach (string input in Arguments) foreach (string input in Arguments)
{ {
DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: hashes);
} }
// TODO: We might not want to remove dates in the future // TODO: We might not want to remove dates in the future

5
SabreTools/Features/Verify.cs
View File

@@ -53,6 +53,7 @@ namespace SabreTools.Features
TreatAsFile asFiles = GetTreatAsFiles(features); TreatAsFile asFiles = GetTreatAsFiles(features);
bool hashOnly = GetBoolean(features, HashOnlyValue); bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue); bool quickScan = GetBoolean(features, QuickValue);
HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1];
// If we are in individual mode, process each DAT on their own // If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue)) if (GetBoolean(features, IndividualValue))
@@ -91,7 +92,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n"); logger.User("Processing files:\n");
foreach (string input in Inputs) foreach (string input in Inputs)
{ {
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
} }
Verification.VerifyGeneric(datdata, hashOnly); Verification.VerifyGeneric(datdata, hashOnly);
@@ -145,7 +146,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n"); logger.User("Processing files:\n");
foreach (string input in Inputs) foreach (string input in Inputs)
{ {
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]); DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
} }
Verification.VerifyGeneric(datdata, hashOnly); Verification.VerifyGeneric(datdata, hashOnly);