claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make DFD code cleaner for future work

Browse Source
This commit is contained in:
Matt Nadareski
2024-10-24 05:52:48 -04:00
parent 15e30d7a75
commit dd06c076d7
8 changed files with 66 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
RombaSharp/Features/Archive.cs
View File

@@ -59,6 +59,7 @@ have a current entry in the DAT index.";
bool noDb = GetBoolean(features, NoDbValue); bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue); bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// First we want to get just all directories from the inputs // First we want to get just all directories from the inputs
List<string> onlyDirs = []; List<string> onlyDirs = [];
@@ -72,8 +73,8 @@ have a current entry in the DAT index.";
DatFile df = DatFile.Create(); DatFile df = DatFile.Create();
foreach (string dir in onlyDirs) foreach (string dir in onlyDirs)
{ {
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: hashes); dfd.PopulateFromDir(df, dir, TreatAsFile.NonArchive);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: hashes); dfd.PopulateFromDir(df, dir, TreatAsFile.All);
} }
// Create an empty Dat for files that need to be rebuilt // Create an empty Dat for files that need to be rebuilt

3
RombaSharp/Features/Dir2Dat.cs
View File

@@ -42,6 +42,7 @@ namespace RombaSharp.Features
string? source = GetString(features, SourceStringValue); string? source = GetString(features, SourceStringValue);
string? outdat = GetString(features, OutStringValue); string? outdat = GetString(features, OutStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Ensure the output directory // Ensure the output directory
outdat = outdat.Ensure(create: true); outdat = outdat.Ensure(create: true);
@@ -57,7 +58,7 @@ namespace RombaSharp.Features
DatFile datfile = DatFile.Create(); DatFile datfile = DatFile.Create();
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name); datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name);
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description); datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description);
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: hashes); dfd.PopulateFromDir(datfile, source, TreatAsFile.NonArchive);
Writer.Write(datfile, outdat!); Writer.Write(datfile, outdat!);
return true; return true;
} }

3
RombaSharp/Features/RefreshDats.cs
View File

@@ -45,6 +45,7 @@ contents of any changed dats.";
int workers = GetInt32(features, WorkersInt32Value); int workers = GetInt32(features, WorkersInt32Value);
string? missingSha1s = GetString(features, MissingSha1sStringValue); string? missingSha1s = GetString(features, MissingSha1sStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Make sure the db is set // Make sure the db is set
if (string.IsNullOrWhiteSpace(_db)) if (string.IsNullOrWhiteSpace(_db))
@@ -70,7 +71,7 @@ contents of any changed dats.";
// First get a list of SHA-1's from the input DATs // First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create(); DatFile datroot = DatFile.Create();
datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT"); datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT");
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: hashes); dfd.PopulateFromDir(datroot, _dats, TreatAsFile.NonArchive);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1) // Create a List of dat hashes in the database (SHA-1)

3
RombaSharp/Features/RescanDepots.cs
View File

@@ -36,6 +36,7 @@ namespace RombaSharp.Features
return false; return false;
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
logger.Error("This feature is not yet implemented: rescan-depots"); logger.Error("This feature is not yet implemented: rescan-depots");
@@ -74,7 +75,7 @@ namespace RombaSharp.Features
// Now rescan the depot itself // Now rescan the depot itself
DatFile depot = DatFile.Create(); DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: hashes); dfd.PopulateFromDir(depot, depotname, TreatAsFile.NonArchive);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use // Set the base queries to use

95
SabreTools.DatTools/DatFromDir.cs
View File

@@ -4,9 +4,6 @@ using System.Threading;
#if NET40_OR_GREATER || NETCOREAPP #if NET40_OR_GREATER || NETCOREAPP
using System.Threading.Tasks; using System.Threading.Tasks;
#endif #endif
#if NET452_OR_GREATER || NETCOREAPP
using SabreTools.Core;
#endif
using SabreTools.Core.Tools; using SabreTools.Core.Tools;
using SabreTools.DatFiles; using SabreTools.DatFiles;
using SabreTools.DatItems; using SabreTools.DatItems;
@@ -25,6 +22,25 @@ namespace SabreTools.DatTools
/// </summary> /// </summary>
public class DatFromDir public class DatFromDir
{ {
#region Fields
/// <summary>
/// Hashes to include in the information
/// </summary>
private readonly HashType[] _hashes;
/// <summary>
/// Type of files that should be skipped
/// </summary>
private readonly SkipFileType _skipFileType;
/// <summary>
/// Indicates if blank items should be created for empty folders
/// </summary>
private readonly bool _addBlanks;
#endregion
#region Logging #region Logging
/// <summary> /// <summary>
@@ -34,26 +50,25 @@ namespace SabreTools.DatTools
#endregion #endregion
#region Constructors
public DatFromDir(HashType[]? hashes, SkipFileType skipFileType, bool addBlanks)
{
_hashes = hashes ?? [HashType.CRC32, HashType.MD5, HashType.SHA1];
_skipFileType = skipFileType;
_addBlanks = addBlanks;
}
#endregion
/// <summary> /// <summary>
/// Create a new Dat from a directory /// Create a new Dat from a directory
/// </summary> /// </summary>
/// <param name="datFile">Current DatFile object to add to</param> /// <param name="datFile">Current DatFile object to add to</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param> /// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param> public bool PopulateFromDir(DatFile datFile, string basePath, TreatAsFile asFiles = 0x00)
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
public static bool PopulateFromDir(
DatFile datFile,
string basePath,
TreatAsFile asFiles = 0x00,
SkipFileType skipFileType = SkipFileType.None,
bool addBlanks = false,
HashType[]? hashes = null)
{ {
// If no hashes are set, use the standard array
hashes ??= [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Set the progress variables // Set the progress variables
long totalSize = 0; long totalSize = 0;
long currentSize = 0; long currentSize = 0;
@@ -74,7 +89,7 @@ namespace SabreTools.DatTools
// Loop through and add the file sizes // Loop through and add the file sizes
#if NET452_OR_GREATER || NETCOREAPP #if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(files, Globals.ParallelOptions, item => Parallel.ForEach(files, Core.Globals.ParallelOptions, item =>
#elif NET40_OR_GREATER #elif NET40_OR_GREATER
Parallel.ForEach(files, item => Parallel.ForEach(files, item =>
#else #else
@@ -94,12 +109,12 @@ namespace SabreTools.DatTools
{ {
currentSize += new FileInfo(item).Length; currentSize += new FileInfo(item).Length;
CheckFileForHashes(datFile, item, basePath, asFiles, skipFileType, addBlanks, hashes); CheckFileForHashes(datFile, item, basePath, asFiles);
logger.User(totalSize, currentSize, item); logger.User(totalSize, currentSize, item);
} }
// Now find all folders that are empty, if we are supposed to // Now find all folders that are empty, if we are supposed to
if (addBlanks) if (_addBlanks)
ProcessDirectoryBlanks(datFile, basePath); ProcessDirectoryBlanks(datFile, basePath);
} }
else if (System.IO.File.Exists(basePath)) else if (System.IO.File.Exists(basePath))
@@ -110,7 +125,7 @@ namespace SabreTools.DatTools
logger.User(totalSize, currentSize); logger.User(totalSize, currentSize);
string? parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath)); string? parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(datFile, basePath, parentPath, asFiles, skipFileType, addBlanks, hashes); CheckFileForHashes(datFile, basePath, parentPath, asFiles);
logger.User(totalSize, totalSize, basePath); logger.User(totalSize, totalSize, basePath);
} }
@@ -125,17 +140,7 @@ namespace SabreTools.DatTools
/// <param name="item">Filename of the item to be checked</param> /// <param name="item">Filename of the item to be checked</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param> /// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param> private void CheckFileForHashes(DatFile datFile, string item, string? basePath, TreatAsFile asFiles)
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
private static void CheckFileForHashes(
DatFile datFile,
string item,
string? basePath,
TreatAsFile asFiles,
SkipFileType skipFileType,
bool addBlanks,
HashType[] hashes)
{ {
// If we're in depot mode, process it separately // If we're in depot mode, process it separately
if (CheckDepotFile(datFile, item)) if (CheckDepotFile(datFile, item))
@@ -148,20 +153,20 @@ namespace SabreTools.DatTools
if (archive != null) if (archive != null)
{ {
// Set the archive flags // Set the archive flags
archive.AvailableHashTypes = hashes; archive.AvailableHashTypes = _hashes;
// Skip if we're treating archives as files and skipping files // Skip if we're treating archives as files and skipping files
#if NETFRAMEWORK #if NETFRAMEWORK
if ((asFiles & TreatAsFile.Archive) != 0 && skipFileType == SkipFileType.File) if ((asFiles & TreatAsFile.Archive) != 0 && _skipFileType == SkipFileType.File)
#else #else
if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File) if (asFiles.HasFlag(TreatAsFile.Archive) && _skipFileType == SkipFileType.File)
#endif #endif
{ {
return; return;
} }
// Skip if we're skipping archives // Skip if we're skipping archives
else if (skipFileType == SkipFileType.Archive) else if (_skipFileType == SkipFileType.Archive)
{ {
return; return;
} }
@@ -180,14 +185,14 @@ namespace SabreTools.DatTools
ProcessArchive(datFile, item, basePath, extracted); ProcessArchive(datFile, item, basePath, extracted);
// Now find all folders that are empty, if we are supposed to // Now find all folders that are empty, if we are supposed to
if (addBlanks) if (_addBlanks)
ProcessArchiveBlanks(datFile, item, basePath, archive); ProcessArchiveBlanks(datFile, item, basePath, archive);
} }
// Process as file if we're treating archives as files // Process as file if we're treating archives as files
else else
{ {
ProcessFile(datFile, item, basePath, hashes, asFiles); ProcessFile(datFile, item, basePath, asFiles);
} }
} }
@@ -195,12 +200,12 @@ namespace SabreTools.DatTools
else else
{ {
// Skip if we're skipping files // Skip if we're skipping files
if (skipFileType == SkipFileType.File) if (_skipFileType == SkipFileType.File)
return; return;
// Process as file // Process as file
else else
ProcessFile(datFile, item, basePath, hashes, asFiles); ProcessFile(datFile, item, basePath, asFiles);
} }
} }
@@ -251,7 +256,7 @@ namespace SabreTools.DatTools
// First take care of the found items // First take care of the found items
#if NET452_OR_GREATER || NETCOREAPP #if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile => Parallel.ForEach(extracted, Core.Globals.ParallelOptions, baseFile =>
#elif NET40_OR_GREATER #elif NET40_OR_GREATER
Parallel.ForEach(extracted, baseFile => Parallel.ForEach(extracted, baseFile =>
#else #else
@@ -294,7 +299,7 @@ namespace SabreTools.DatTools
// Add add all of the found empties to the DAT // Add add all of the found empties to the DAT
#if NET452_OR_GREATER || NETCOREAPP #if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Globals.ParallelOptions, empty => Parallel.ForEach(empties, Core.Globals.ParallelOptions, empty =>
#elif NET40_OR_GREATER #elif NET40_OR_GREATER
Parallel.ForEach(empties, empty => Parallel.ForEach(empties, empty =>
#else #else
@@ -329,7 +334,7 @@ namespace SabreTools.DatTools
List<string> empties = basePath.ListEmpty() ?? []; List<string> empties = basePath.ListEmpty() ?? [];
#if NET452_OR_GREATER || NETCOREAPP #if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Globals.ParallelOptions, dir => Parallel.ForEach(empties, Core.Globals.ParallelOptions, dir =>
#elif NET40_OR_GREATER #elif NET40_OR_GREATER
Parallel.ForEach(empties, dir => Parallel.ForEach(empties, dir =>
#else #else
@@ -396,12 +401,12 @@ namespace SabreTools.DatTools
/// <param name="datFile">Current DatFile object to add to</param> /// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param> /// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param> /// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string? basePath, HashType[] hashes, TreatAsFile asFiles) private void ProcessFile(DatFile datFile, string item, string? basePath, TreatAsFile asFiles)
{ {
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile? baseFile = BaseFile.GetInfo(item, header: datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey), hashes: hashes, asFiles: asFiles); var header = datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey);
BaseFile? baseFile = BaseFile.GetInfo(item, header, _hashes, asFiles);
DatItem? datItem = DatItem.Create(baseFile); DatItem? datItem = DatItem.Create(baseFile);
if (datItem != null) if (datItem != null)
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty); ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);

3
SabreTools/Features/Batch.cs
View File

@@ -279,11 +279,12 @@ Reset the internal state: reset();";
public override void Process(BatchState batchState) public override void Process(BatchState batchState)
{ {
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatTools.DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Assume there could be multiple // Assume there could be multiple
foreach (string input in Arguments) foreach (string input in Arguments)
{ {
DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: hashes); dfd.PopulateFromDir(batchState.DatFile, input);
} }
// TODO: We might not want to remove dates in the future // TODO: We might not want to remove dates in the future

10
SabreTools/Features/DatFromDir.cs
View File

@@ -64,6 +64,7 @@ namespace SabreTools.Features
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue); bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features); var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features); var skipFileType = GetSkipFileType(features);
var dfd = new DatTools.DatFromDir(includeInScan, skipFileType, addBlankFiles);
// Apply the specialized field removals to the cleaner // Apply the specialized field removals to the cleaner
if (!addFileDates) if (!addFileDates)
@@ -90,14 +91,7 @@ namespace SabreTools.Features
datdata.FillHeaderFromPath(basePath, noAutomaticDate); datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path // Now populate from the path
bool success = DatTools.DatFromDir.PopulateFromDir( bool success = dfd.PopulateFromDir(datdata, basePath, asFiles);
datdata,
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
if (success) if (success)
{ {
// Perform additional processing steps // Perform additional processing steps

5
SabreTools/Features/Verify.cs
View File

@@ -54,6 +54,7 @@ namespace SabreTools.Features
bool hashOnly = GetBoolean(features, HashOnlyValue); bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue); bool quickScan = GetBoolean(features, QuickValue);
HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1]; HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatTools.DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// If we are in individual mode, process each DAT on their own // If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue)) if (GetBoolean(features, IndividualValue))
@@ -92,7 +93,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n"); logger.User("Processing files:\n");
foreach (string input in Inputs) foreach (string input in Inputs)
{ {
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes); dfd.PopulateFromDir(datdata, input, asFiles);
} }
Verification.VerifyGeneric(datdata, hashOnly); Verification.VerifyGeneric(datdata, hashOnly);
@@ -146,7 +147,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n"); logger.User("Processing files:\n");
foreach (string input in Inputs) foreach (string input in Inputs)
{ {
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes); dfd.PopulateFromDir(datdata, input, asFiles);
} }
Verification.VerifyGeneric(datdata, hashOnly); Verification.VerifyGeneric(datdata, hashOnly);