claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make DFD code cleaner for future work

Browse Source
This commit is contained in:
Matt Nadareski
2024-10-24 05:52:48 -04:00
parent 15e30d7a75
commit dd06c076d7
8 changed files with 66 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
RombaSharp/Features/Archive.cs
View File

@@ -59,6 +59,7 @@ have a current entry in the DAT index.";
bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// First we want to get just all directories from the inputs
List<string> onlyDirs = [];
@@ -72,8 +73,8 @@ have a current entry in the DAT index.";
DatFile df = DatFile.Create();
foreach (string dir in onlyDirs)
{
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: hashes);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: hashes);
dfd.PopulateFromDir(df, dir, TreatAsFile.NonArchive);
dfd.PopulateFromDir(df, dir, TreatAsFile.All);
}
// Create an empty Dat for files that need to be rebuilt

3
RombaSharp/Features/Dir2Dat.cs
View File

@@ -42,6 +42,7 @@ namespace RombaSharp.Features
string? source = GetString(features, SourceStringValue);
string? outdat = GetString(features, OutStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Ensure the output directory
outdat = outdat.Ensure(create: true);
@@ -57,7 +58,7 @@ namespace RombaSharp.Features
DatFile datfile = DatFile.Create();
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.NameKey, string.IsNullOrWhiteSpace(name) ? "untitled" : name);
datfile.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.DescriptionKey, description);
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: hashes);
dfd.PopulateFromDir(datfile, source, TreatAsFile.NonArchive);
Writer.Write(datfile, outdat!);
return true;
}

3
RombaSharp/Features/RefreshDats.cs
View File

@@ -45,6 +45,7 @@ contents of any changed dats.";
int workers = GetInt32(features, WorkersInt32Value);
string? missingSha1s = GetString(features, MissingSha1sStringValue);
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Make sure the db is set
if (string.IsNullOrWhiteSpace(_db))
@@ -70,7 +71,7 @@ contents of any changed dats.";
// First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create();
datroot.Header.SetFieldValue<string?>(SabreTools.Models.Metadata.Header.TypeKey, "SuperDAT");
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: hashes);
dfd.PopulateFromDir(datroot, _dats, TreatAsFile.NonArchive);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)

3
RombaSharp/Features/RescanDepots.cs
View File

@@ -36,6 +36,7 @@ namespace RombaSharp.Features
return false;
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatFromDir(hashes, SkipFileType.None, addBlanks: false);
logger.Error("This feature is not yet implemented: rescan-depots");
@@ -74,7 +75,7 @@ namespace RombaSharp.Features
// Now rescan the depot itself
DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: hashes);
dfd.PopulateFromDir(depot, depotname, TreatAsFile.NonArchive);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use

95
SabreTools.DatTools/DatFromDir.cs
View File

@@ -4,9 +4,6 @@ using System.Threading;
#if NET40_OR_GREATER || NETCOREAPP
using System.Threading.Tasks;
#endif
#if NET452_OR_GREATER || NETCOREAPP
using SabreTools.Core;
#endif
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
@@ -25,6 +22,25 @@ namespace SabreTools.DatTools
/// </summary>
public class DatFromDir
{
#region Fields
/// <summary>
/// Hashes to include in the information
/// </summary>
private readonly HashType[] _hashes;
/// <summary>
/// Type of files that should be skipped
/// </summary>
private readonly SkipFileType _skipFileType;
/// <summary>
/// Indicates if blank items should be created for empty folders
/// </summary>
private readonly bool _addBlanks;
#endregion
#region Logging
/// <summary>
@@ -34,26 +50,25 @@ namespace SabreTools.DatTools
#endregion
#region Constructors
public DatFromDir(HashType[]? hashes, SkipFileType skipFileType, bool addBlanks)
{
_hashes = hashes ?? [HashType.CRC32, HashType.MD5, HashType.SHA1];
_skipFileType = skipFileType;
_addBlanks = addBlanks;
}
#endregion
/// <summary>
/// Create a new Dat from a directory
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
public static bool PopulateFromDir(
DatFile datFile,
string basePath,
TreatAsFile asFiles = 0x00,
SkipFileType skipFileType = SkipFileType.None,
bool addBlanks = false,
HashType[]? hashes = null)
public bool PopulateFromDir(DatFile datFile, string basePath, TreatAsFile asFiles = 0x00)
{
// If no hashes are set, use the standard array
hashes ??= [HashType.CRC32, HashType.MD5, HashType.SHA1];
// Set the progress variables
long totalSize = 0;
long currentSize = 0;
@@ -74,7 +89,7 @@ namespace SabreTools.DatTools
// Loop through and add the file sizes
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(files, Globals.ParallelOptions, item =>
Parallel.ForEach(files, Core.Globals.ParallelOptions, item =>
#elif NET40_OR_GREATER
Parallel.ForEach(files, item =>
#else
@@ -94,12 +109,12 @@ namespace SabreTools.DatTools
{
currentSize += new FileInfo(item).Length;
CheckFileForHashes(datFile, item, basePath, asFiles, skipFileType, addBlanks, hashes);
CheckFileForHashes(datFile, item, basePath, asFiles);
logger.User(totalSize, currentSize, item);
}
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
if (_addBlanks)
ProcessDirectoryBlanks(datFile, basePath);
}
else if (System.IO.File.Exists(basePath))
@@ -110,7 +125,7 @@ namespace SabreTools.DatTools
logger.User(totalSize, currentSize);
string? parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(datFile, basePath, parentPath, asFiles, skipFileType, addBlanks, hashes);
CheckFileForHashes(datFile, basePath, parentPath, asFiles);
logger.User(totalSize, totalSize, basePath);
}
@@ -125,17 +140,7 @@ namespace SabreTools.DatTools
/// <param name="item">Filename of the item to be checked</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
private static void CheckFileForHashes(
DatFile datFile,
string item,
string? basePath,
TreatAsFile asFiles,
SkipFileType skipFileType,
bool addBlanks,
HashType[] hashes)
private void CheckFileForHashes(DatFile datFile, string item, string? basePath, TreatAsFile asFiles)
{
// If we're in depot mode, process it separately
if (CheckDepotFile(datFile, item))
@@ -148,20 +153,20 @@ namespace SabreTools.DatTools
if (archive != null)
{
// Set the archive flags
archive.AvailableHashTypes = hashes;
archive.AvailableHashTypes = _hashes;
// Skip if we're treating archives as files and skipping files
#if NETFRAMEWORK
if ((asFiles & TreatAsFile.Archive) != 0 && skipFileType == SkipFileType.File)
if ((asFiles & TreatAsFile.Archive) != 0 && _skipFileType == SkipFileType.File)
#else
if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File)
if (asFiles.HasFlag(TreatAsFile.Archive) && _skipFileType == SkipFileType.File)
#endif
{
return;
}
// Skip if we're skipping archives
else if (skipFileType == SkipFileType.Archive)
else if (_skipFileType == SkipFileType.Archive)
{
return;
}
@@ -180,14 +185,14 @@ namespace SabreTools.DatTools
ProcessArchive(datFile, item, basePath, extracted);
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
if (_addBlanks)
ProcessArchiveBlanks(datFile, item, basePath, archive);
}
// Process as file if we're treating archives as files
else
{
ProcessFile(datFile, item, basePath, hashes, asFiles);
ProcessFile(datFile, item, basePath, asFiles);
}
}
@@ -195,12 +200,12 @@ namespace SabreTools.DatTools
else
{
// Skip if we're skipping files
if (skipFileType == SkipFileType.File)
if (_skipFileType == SkipFileType.File)
return;
// Process as file
else
ProcessFile(datFile, item, basePath, hashes, asFiles);
ProcessFile(datFile, item, basePath, asFiles);
}
}
@@ -251,7 +256,7 @@ namespace SabreTools.DatTools
// First take care of the found items
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile =>
Parallel.ForEach(extracted, Core.Globals.ParallelOptions, baseFile =>
#elif NET40_OR_GREATER
Parallel.ForEach(extracted, baseFile =>
#else
@@ -294,7 +299,7 @@ namespace SabreTools.DatTools
// Add add all of the found empties to the DAT
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Globals.ParallelOptions, empty =>
Parallel.ForEach(empties, Core.Globals.ParallelOptions, empty =>
#elif NET40_OR_GREATER
Parallel.ForEach(empties, empty =>
#else
@@ -329,7 +334,7 @@ namespace SabreTools.DatTools
List<string> empties = basePath.ListEmpty() ?? [];
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Globals.ParallelOptions, dir =>
Parallel.ForEach(empties, Core.Globals.ParallelOptions, dir =>
#elif NET40_OR_GREATER
Parallel.ForEach(empties, dir =>
#else
@@ -396,12 +401,12 @@ namespace SabreTools.DatTools
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string? basePath, HashType[] hashes, TreatAsFile asFiles)
private void ProcessFile(DatFile datFile, string item, string? basePath, TreatAsFile asFiles)
{
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile? baseFile = BaseFile.GetInfo(item, header: datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey), hashes: hashes, asFiles: asFiles);
var header = datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey);
BaseFile? baseFile = BaseFile.GetInfo(item, header, _hashes, asFiles);
DatItem? datItem = DatItem.Create(baseFile);
if (datItem != null)
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);

3
SabreTools/Features/Batch.cs
View File

@@ -279,11 +279,12 @@ Reset the internal state: reset();";
public override void Process(BatchState batchState)
{
HashType[] hashes = [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatTools.DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// Assume there could be multiple
foreach (string input in Arguments)
{
DatTools.DatFromDir.PopulateFromDir(batchState.DatFile, input, hashes: hashes);
dfd.PopulateFromDir(batchState.DatFile, input);
}
// TODO: We might not want to remove dates in the future

10
SabreTools/Features/DatFromDir.cs
View File

@@ -64,6 +64,7 @@ namespace SabreTools.Features
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
var dfd = new DatTools.DatFromDir(includeInScan, skipFileType, addBlankFiles);
// Apply the specialized field removals to the cleaner
if (!addFileDates)
@@ -90,14 +91,7 @@ namespace SabreTools.Features
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = DatTools.DatFromDir.PopulateFromDir(
datdata,
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
bool success = dfd.PopulateFromDir(datdata, basePath, asFiles);
if (success)
{
// Perform additional processing steps

5
SabreTools/Features/Verify.cs
View File

@@ -54,6 +54,7 @@ namespace SabreTools.Features
bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue);
HashType[] hashes = quickScan ? [HashType.CRC32] : [HashType.CRC32, HashType.MD5, HashType.SHA1];
var dfd = new DatTools.DatFromDir(hashes, SkipFileType.None, addBlanks: false);
// If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue))
@@ -92,7 +93,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
dfd.PopulateFromDir(datdata, input, asFiles);
}
Verification.VerifyGeneric(datdata, hashOnly);
@@ -146,7 +147,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: hashes);
dfd.PopulateFromDir(datdata, input, asFiles);
}
Verification.VerifyGeneric(datdata, hashOnly);