claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce array allocations

Browse Source
This commit is contained in:
Matt Nadareski
2024-10-24 05:33:28 -04:00
parent e992f0d013
commit 15e30d7a75
6 changed files with 21 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
RombaSharp/Features/Archive.cs
View File

@@ -58,6 +58,7 @@ have a current entry in the DAT index.";
// Get feature flags
bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// First we want to get just all directories from the inputs
List<string> onlyDirs = [];
@@ -71,8 +72,8 @@ have a current entry in the DAT index.";
DatFile df = DatFile.Create();
foreach (string dir in onlyDirs)
{
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: hashes);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: hashes);
}
// Create an empty Dat for files that need to be rebuilt

3
RombaSharp/Features/Dir2Dat.cs
View File

@@ -41,6 +41,7 @@ namespace RombaSharp.Features
string? description = GetString(features, DescriptionStringValue);
string? source = GetString(features, SourceStringValue);
string? outdat = GetString(features, OutStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Ensure the output directory
outdat = outdat.Ensure(create: true);
@@ -56,7 +57,7 @@ namespace RombaSharp.Features
DatFile datfile = DatFile.Create();
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name);
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description);
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: hashes);
Writer.Write(datfile, outdat!);
return true;
}

3
RombaSharp/Features/RefreshDats.cs
View File

@@ -44,6 +44,7 @@ contents of any changed dats.";
// Get feature flags
int workers = GetInt32(features, WorkersInt32Value);
string? missingSha1s = GetString(features, MissingSha1sStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Make sure the db is set
if (string.IsNullOrWhiteSpace(_db))
@@ -69,7 +70,7 @@ contents of any changed dats.";
// First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create();
datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT");
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: hashes);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)

14
RombaSharp/Features/RescanDepots.cs
View File

@@ -35,6 +35,8 @@ namespace RombaSharp.Features
if (!base.ProcessFeatures(features))
return false;
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
logger.Error("This feature is not yet implemented: rescan-depots");
foreach (string depotname in Inputs)
@@ -58,7 +60,7 @@ namespace RombaSharp.Features
dbc.Open();
// If we have it, then check for all hashes that are in that depot
List<string> hashes = [];
List<string> sha1Hashes = [];
string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
@@ -66,13 +68,13 @@ namespace RombaSharp.Features
{
while (sldr.Read())
{
hashes.Add(sldr.GetString(0));
sha1Hashes.Add(sldr.GetString(0));
}
}
// Now rescan the depot itself
DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: hashes);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use
@@ -93,10 +95,10 @@ namespace RombaSharp.Features
foreach (Rom rom in roms)
{
if (hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!))
if (sha1Hashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!))
{
dupehashes.Add(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!);
hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!);
sha1Hashes.Remove(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!);
}
else if (!dupehashes.Contains(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)!))
{
@@ -162,7 +164,7 @@ JOIN crc
JOIN md5
ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN ";
query += $"({string.Join("\",\"", hashes)}\")";
query += $"({string.Join("\",\"", sha1Hashes)}\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();

4
SabreTools/Features/Batch.cs
View File

@@ -278,10 +278,12 @@ Reset the internal state: reset();";
/// <remarks>TODO: Should any of the other options be added for D2D?</remarks>
public override void Process(BatchState batchState)
{
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Assume there could be multiple
foreach (string input in Arguments)
{
DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: hashes);
}
// TODO: We might not want to remove dates in the future

5
SabreTools/Features/Verify.cs
View File

@@ -53,6 +53,7 @@ namespace SabreTools.Features
TreatAsFile asFiles = GetTreatAsFiles(features);
bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue);
HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1];
// If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue))
@@ -91,7 +92,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
}
Verification.VerifyGeneric(datdata, hashOnly);
@@ -145,7 +146,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]);
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
}
Verification.VerifyGeneric(datdata, hashOnly);