claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix hashing in parallel

Browse Source
This commit is contained in:
Matt Nadareski
2020-10-05 17:43:44 -07:00
parent 88f69442df
commit 982df3faaf
17 changed files with 167 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
RombaSharp/Features/BaseFeature.cs
View File

@@ -394,7 +394,7 @@ namespace RombaSharp.Features
if (lowerCaseDats.Contains(input.ToLowerInvariant()))
{
string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]);
string sha1 = Utilities.ByteArrayToString(FileExtensions.GetInfo(fullpath).SHA1);
string sha1 = Utilities.ByteArrayToString(FileExtensions.GetInfo(fullpath, hashes: Hash.SHA1).SHA1);
foundDats.Add(sha1, fullpath);
}
else

32
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -1974,13 +1974,13 @@ namespace SabreTools.Library.DatFiles
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="quickScan">True if archive header should be used, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
public bool PopulateFromDir(
string basePath,
TreatAsFile asFiles = 0x00,
SkipFileType skipFileType = SkipFileType.None,
bool addBlanks = false,
bool quickScan = false)
Hash hashes = Hash.Standard)
{
// Clean the temp directory path
Globals.TempDir = DirectoryExtensions.Ensure(Globals.TempDir, temp: true);
@@ -1994,7 +1994,7 @@ namespace SabreTools.Library.DatFiles
List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(files, Globals.ParallelOptions, item =>
{
CheckFileForHashes(item, basePath, asFiles, skipFileType, addBlanks, quickScan);
CheckFileForHashes(item, basePath, asFiles, skipFileType, addBlanks, hashes);
});
// Now find all folders that are empty, if we are supposed to
@@ -2004,7 +2004,7 @@ namespace SabreTools.Library.DatFiles
else if (File.Exists(basePath))
{
string parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(basePath, parentPath, asFiles, skipFileType, addBlanks, quickScan);
CheckFileForHashes(basePath, parentPath, asFiles, skipFileType, addBlanks, hashes);
}
// Now that we're done, delete the temp folder (if it's not the default)
@@ -2023,19 +2023,23 @@ namespace SabreTools.Library.DatFiles
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="quickScan">True if archive header should be used, false otherwise</param>
private void CheckFileForHashes(string item, string basePath, TreatAsFile asFiles, SkipFileType skipFileType, bool addBlanks, bool quickScan)
/// <param name="hashes">Hashes to include in the information</param>
private void CheckFileForHashes(string item, string basePath, TreatAsFile asFiles, SkipFileType skipFileType, bool addBlanks, Hash hashes)
{
// If we're in depot mode, process it separately
if (CheckDepotFile(item))
return;
// Initialize possible archive variables
BaseArchive archive = BaseArchive.Create(item, quickScan);
BaseArchive archive = BaseArchive.Create(item);
// Process archives according to flags
if (archive != null)
{
// Set the archive flags
archive.AvailableHashes = hashes;
archive.QuickScan = hashes == Hash.CRC;
// Skip if we're treating archives as files and skipping files
if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File)
{
@@ -2065,7 +2069,7 @@ namespace SabreTools.Library.DatFiles
// Process as file if we're treating archives as files
else
{
ProcessFile(item, basePath, asFiles);
ProcessFile(item, basePath, hashes, asFiles);
}
}
@@ -2078,7 +2082,7 @@ namespace SabreTools.Library.DatFiles
// Process as file
else
ProcessFile(item, basePath, asFiles);
ProcessFile(item, basePath, hashes, asFiles);
}
}
@@ -2206,11 +2210,12 @@ namespace SabreTools.Library.DatFiles
/// </summary>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private void ProcessFile(string item, string basePath, TreatAsFile asFiles)
private void ProcessFile(string item, string basePath, Hash hashes, TreatAsFile asFiles)
{
Globals.Logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile baseFile = FileExtensions.GetInfo(item, header: Header.HeaderSkipper, asFiles: asFiles);
BaseFile baseFile = FileExtensions.GetInfo(item, header: Header.HeaderSkipper, hashes: hashes, asFiles: asFiles);
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(item, datItem, basePath, string.Empty);
}
@@ -2604,11 +2609,14 @@ namespace SabreTools.Library.DatFiles
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file, quickScan);
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.QuickScan = quickScan;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))

8
SabreTools.Library/FileTypes/BaseArchive.cs
View File

@@ -55,10 +55,8 @@ namespace SabreTools.Library.FileTypes
/// Create an archive object from a filename, if possible
/// </summary>
/// <param name="input">Name of the file to create the archive from</param>
/// <param name="quickScan">True to use archive header values, false otherwise</param>
/// <param name="useDates">True to use dates for read and write, false otherwise</param>
/// <returns>Archive object representing the inputs</returns>
public static BaseArchive Create(string input, bool quickScan = false, bool useDates = false)
public static BaseArchive Create(string input)
{
BaseArchive archive = null;
@@ -98,10 +96,6 @@ namespace SabreTools.Library.FileTypes
break;
}
// Set the quickscan flag
if (archive != null)
archive.QuickScan = quickScan;
return archive;
}

81
SabreTools.Library/FileTypes/BaseFile.cs
View File

@@ -1,4 +1,5 @@
using System.IO;
using SabreTools.Library.DatFiles;
using SabreTools.Library.IO;
@@ -6,24 +7,80 @@ namespace SabreTools.Library.FileTypes
{
public class BaseFile
{
#region Publicly facing variables
// TODO: Get all of these values automatically so there is no public "set"
#region Fields
/// <summary>
/// Internal type of the represented file
/// </summary>
public FileType Type { get; protected set; }
/// <summary>
/// Filename or path to the file
/// </summary>
public string Filename { get; set; }
/// <summary>
/// Direct parent of the file
/// </summary>
public string Parent { get; set; }
/// <summary>
/// Date stamp of the file
/// </summary>
public string Date { get; set; }
/// <summary>
/// Optional size of the file
/// </summary>
public long? Size { get; set; }
public byte[] CRC { get; set; }
public byte[] MD5 { get; set; }
/// <summary>
/// Hashes that are available for the file
/// </summary>
public Hash AvailableHashes { get; set; } = Hash.Standard;
/// <summary>
/// CRC32 hash of the file
/// </summary>
public byte[] CRC { get; set; } = null;
/// <summary>
/// MD5 hash of the file
/// </summary>
public byte[] MD5 { get; set; } = null;
#if NET_FRAMEWORK
public byte[] RIPEMD160 { get; set; }
/// <summary>
/// RIPEMD160 hash of the file
/// </summary>
public byte[] RIPEMD160 { get; set; } = null;
#endif
public byte[] SHA1 { get; set; }
public byte[] SHA256 { get; set; }
public byte[] SHA384 { get; set; }
public byte[] SHA512 { get; set; }
public byte[] SpamSum { get; set; }
/// <summary>
/// SHA-1 hash of the file
/// </summary>
public byte[] SHA1 { get; set; } = null;
/// <summary>
/// SHA-256 hash of the file
/// </summary>
public byte[] SHA256 { get; set; } = null;
/// <summary>
/// SHA-384 hash of the file
/// </summary>
public byte[] SHA384 { get; set; } = null;
/// <summary>
/// SHA-512 hash of the file
/// </summary>
public byte[] SHA512 { get; set; } = null;
/// <summary>
/// SpamSum fuzzy hash of the file
/// </summary>
public byte[] SpamSum { get; set; } = null;
#endregion
@@ -47,7 +104,7 @@ namespace SabreTools.Library.FileTypes
if (getHashes)
{
BaseFile temp = FileExtensions.GetInfo(this.Filename);
BaseFile temp = FileExtensions.GetInfo(this.Filename, hashes: this.AvailableHashes);
if (temp != null)
{
this.Parent = temp.Parent;
@@ -78,7 +135,7 @@ namespace SabreTools.Library.FileTypes
if (getHashes)
{
BaseFile temp = stream.GetInfo();
BaseFile temp = stream.GetInfo(hashes: this.AvailableHashes);
if (temp != null)
{
this.Parent = temp.Parent;

2
SabreTools.Library/FileTypes/Folder.cs
View File

@@ -264,7 +264,7 @@ namespace SabreTools.Library.FileTypes
_children = new List<BaseFile>();
foreach (string file in Directory.EnumerateFiles(this.Filename, "*", SearchOption.TopDirectoryOnly))
{
BaseFile nf = FileExtensions.GetInfo(file);
BaseFile nf = FileExtensions.GetInfo(file, hashes: this.AvailableHashes);
_children.Add(nf);
}

24
SabreTools.Library/FileTypes/GZipArchive.cs
View File

@@ -217,20 +217,19 @@ namespace SabreTools.Library.FileTypes
{
try
{
// Create a blank item for the entry
BaseFile gzipEntryRom = new BaseFile();
// Perform a quickscan, if flagged to
if (QuickScan)
{
BaseFile tempRom = new BaseFile()
gzipEntryRom.Filename = gamename;
using (BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename)))
{
Filename = gamename,
};
BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename));
br.BaseStream.Seek(-8, SeekOrigin.End);
tempRom.CRC = br.ReadBytesBigEndian(4);
tempRom.Size = br.ReadInt32BigEndian();
br.Dispose();
_children.Add(tempRom);
gzipEntryRom.CRC = br.ReadBytesBigEndian(4);
gzipEntryRom.Size = br.ReadInt32BigEndian();
}
}
// Otherwise, use the stream directly
else
@@ -238,13 +237,16 @@ namespace SabreTools.Library.FileTypes
var gz = new gZip();
ZipReturn ret = gz.ZipFileOpen(this.Filename);
ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
BaseFile gzipEntryRom = gzstream.GetInfo();
gzipEntryRom = gzstream.GetInfo(hashes: this.AvailableHashes);
gzipEntryRom.Filename = gz.Filename(0);
gzipEntryRom.Parent = gamename;
gzipEntryRom.Date = (gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);
_children.Add(gzipEntryRom);
gzstream.Dispose();
}
// Fill in comon details and add to the list
gzipEntryRom.Parent = gamename;
_children.Add(gzipEntryRom);
}
catch (Exception ex)
{

2
SabreTools.Library/FileTypes/RarArchive.cs
View File

@@ -199,7 +199,7 @@ namespace SabreTools.Library.FileTypes
{
using (Stream entryStream = entry.OpenEntryStream())
{
rarEntryRom = entryStream.GetInfo(size: entry.Size);
rarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
}
}

2
SabreTools.Library/FileTypes/SevenZipArchive.cs
View File

@@ -307,7 +307,7 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true);
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
}
// Fill in comon details and add to the list

2
SabreTools.Library/FileTypes/TapeArchive.cs
View File

@@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes
{
using (Stream entryStream = entry.OpenEntryStream())
{
tarEntryRom = entryStream.GetInfo(size: entry.Size);
tarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes);
}
}

2
SabreTools.Library/FileTypes/XZArchive.cs
View File

@@ -228,7 +228,7 @@ namespace SabreTools.Library.FileTypes
else
{
var xzStream = new XZStream(File.OpenRead(this.Filename));
BaseFile xzEntryRom = xzStream.GetInfo();
BaseFile xzEntryRom = xzStream.GetInfo(hashes: this.AvailableHashes);
xzEntryRom.Filename = gamename;
xzEntryRom.Parent = gamename;
_children.Add(xzEntryRom);

2
SabreTools.Library/FileTypes/ZipArchive.cs
View File

@@ -308,7 +308,7 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly
else
{
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), keepReadOpen: true);
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
}
// Fill in comon details and add to the list

5
SabreTools.Library/IO/FileExtensions.cs
View File

@@ -340,9 +340,10 @@ namespace SabreTools.Library.IO
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="header">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(string input, string header = null, TreatAsFile asFiles = 0x00)
public static BaseFile GetInfo(string input, string header = null, Hash hashes = Hash.Standard, TreatAsFile asFiles = 0x00)
{
// Add safeguard if file doesn't exist
if (!File.Exists(input))
@@ -376,7 +377,7 @@ namespace SabreTools.Library.IO
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
baseFile = CHDFile.Create(inputStream);
else
baseFile = inputStream.GetInfo(keepReadOpen: false);
baseFile = inputStream.GetInfo(hashes: hashes, keepReadOpen: false);
// Dispose of the input stream
inputStream?.Dispose();

37
SabreTools.Library/IO/StreamExtensions.cs
View File

@@ -43,12 +43,12 @@ namespace SabreTools.Library.IO
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="omitFromScan">Hash flag saying what hashes should not be calculated (defaults to none)</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(this Stream input, long size = -1, bool keepReadOpen = false)
public static BaseFile GetInfo(this Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
{
return GetInfoAsync(input, size, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult();
return GetInfoAsync(input, size, hashes, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult();
}
/// <summary>
@@ -56,9 +56,10 @@ namespace SabreTools.Library.IO
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, bool keepReadOpen = false)
public static async Task<BaseFile> GetInfoAsync(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false)
{
// If we want to automatically set the size
if (size == -1)
@@ -69,15 +70,23 @@ namespace SabreTools.Library.IO
// Get a list of hashers to run over the buffer
List<Hasher> hashers = new List<Hasher>();
if (hashes.HasFlag(Hash.CRC))
hashers.Add(new Hasher(Hash.CRC));
if (hashes.HasFlag(Hash.MD5))
hashers.Add(new Hasher(Hash.MD5));
#if NET_FRAMEWORK
if (hashes.HasFlag(Hash.RIPEMD160))
hashers.Add(new Hasher(Hash.RIPEMD160));
#endif
if (hashes.HasFlag(Hash.SHA1))
hashers.Add(new Hasher(Hash.SHA1));
if (hashes.HasFlag(Hash.SHA256))
hashers.Add(new Hasher(Hash.SHA256));
if (hashes.HasFlag(Hash.SHA384))
hashers.Add(new Hasher(Hash.SHA384));
if (hashes.HasFlag(Hash.SHA512))
hashers.Add(new Hasher(Hash.SHA512));
if (hashes.HasFlag(Hash.SpamSum))
hashers.Add(new Hasher(Hash.SpamSum));
// Initialize the hashing helpers
@@ -112,7 +121,7 @@ namespace SabreTools.Library.IO
byte[] buffer = bufferSelect ? buffer0 : buffer1;
// Run hashes in parallel
Parallel.ForEach(hashers, Globals.ParallelOptions, async h => await h.Process(buffer, current));
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current));
// Wait for the load buffer worker, if needed
if (next > 0)
@@ -126,22 +135,22 @@ namespace SabreTools.Library.IO
// Finalize all hashing helpers
loadBuffer.Finish();
await Task.WhenAll(hashers.Select(h => h.Finalize()));
Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize());
// Get the results
BaseFile baseFile = new BaseFile()
{
Size = size,
CRC = hashers.First(h => h.HashType == Hash.CRC).GetHash(),
MD5 = hashers.First(h => h.HashType == Hash.MD5).GetHash(),
CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null,
MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null,
#if NET_FRAMEWORK
RIPEMD160 = hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash(),
RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null,
#endif
SHA1 = hashers.First(h => h.HashType == Hash.SHA1).GetHash(),
SHA256 = hashers.First(h => h.HashType == Hash.SHA256).GetHash(),
SHA384 = hashers.First(h => h.HashType == Hash.SHA384).GetHash(),
SHA512 = hashers.First(h => h.HashType == Hash.SHA512).GetHash(),
SpamSum = hashers.First(h => h.HashType == Hash.SpamSum).GetHash(),
SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null,
SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null,
SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null,
SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null,
SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null,
};
// Dispose of the hashers

4
SabreTools.Library/Skippers/Transform.cs
View File

@@ -119,7 +119,7 @@ namespace SabreTools.Library.Skippers
// Now add the information to the database if it's not already there
if (!nostore)
{
BaseFile baseFile = FileExtensions.GetInfo(newfile, asFiles: TreatAsFile.NonArchive);
BaseFile baseFile = FileExtensions.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
DatabaseTools.AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile);
}
@@ -139,7 +139,7 @@ namespace SabreTools.Library.Skippers
Directory.CreateDirectory(outDir);
// First, get the SHA-1 hash of the file
BaseFile baseFile = FileExtensions.GetInfo(file, asFiles: TreatAsFile.NonArchive);
BaseFile baseFile = FileExtensions.GetInfo(file, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
// Retrieve a list of all related headers from the database
List<string> headers = DatabaseTools.RetrieveHeadersFromDatabase(Utilities.ByteArrayToString(baseFile.SHA1));

14
SabreTools.Library/Tools/Hasher.cs
View File

@@ -73,12 +73,12 @@ namespace SabreTools.Library.Tools
/// <summary>
/// Process a buffer of some length with the internal hash algorithm
/// </summary>
public async Task Process(byte[] buffer, int size)
public void Process(byte[] buffer, int size)
{
switch (HashType)
{
case Hash.CRC:
await Task.Run(() => (_hasher as OptimizedCRC).Update(buffer, 0, size));
(_hasher as OptimizedCRC).Update(buffer, 0, size);
break;
case Hash.MD5:
@@ -89,11 +89,11 @@ namespace SabreTools.Library.Tools
case Hash.SHA256:
case Hash.SHA384:
case Hash.SHA512:
await Task.Run(() => (_hasher as HashAlgorithm).TransformBlock(buffer, 0, size, null, 0));
(_hasher as HashAlgorithm).TransformBlock(buffer, 0, size, null, 0);
break;
case Hash.SpamSum:
await Task.Run(() => (_hasher as SpamSumContext).Update(buffer));
(_hasher as SpamSumContext).Update(buffer);
break;
}
}
@@ -101,13 +101,13 @@ namespace SabreTools.Library.Tools
/// <summary>
/// Finalize the internal hash algorigthm
/// </summary>
public async Task Finalize()
public void Finalize()
{
byte[] emptyBuffer = new byte[0];
switch (HashType)
{
case Hash.CRC:
await Task.Run(() => (_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0));
(_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0);
break;
case Hash.MD5:
@@ -118,7 +118,7 @@ namespace SabreTools.Library.Tools
case Hash.SHA256:
case Hash.SHA384:
case Hash.SHA512:
await Task.Run(() => (_hasher as HashAlgorithm).TransformFinalBlock(emptyBuffer, 0, 0));
(_hasher as HashAlgorithm).TransformFinalBlock(emptyBuffer, 0, 0);
break;
case Hash.SpamSum:

22
SabreTools/Features/BaseFeature.cs
View File

@@ -2443,30 +2443,30 @@ Some special strings that can be used:
#region Protected Specific Extraction
/// <summary>
/// Get omit from scan from feature list
/// Get include from scan from feature list
/// </summary>
protected Hash GetOmitFromScan(Dictionary<string, Library.Help.Feature> features)
protected Hash GetIncludeInScan(Dictionary<string, Library.Help.Feature> features)
{
Hash omitFromScan = Hash.DeepHashes; // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
Hash includeInScan = Hash.Standard; // TODO: This should be All eventually
if (GetBoolean(features, SkipMd5Value))
omitFromScan |= Hash.MD5;
includeInScan &= ~Hash.MD5;
#if NET_FRAMEWORK
if (GetBoolean(features, SkipRipeMd160Value))
omitFromScan &= ~Hash.RIPEMD160; // TODO: This needs to be inverted later
includeInScan |= Hash.RIPEMD160; // TODO: This needs to be inverted later
#endif
if (GetBoolean(features, SkipSha1Value))
omitFromScan |= Hash.SHA1;
includeInScan &= ~Hash.SHA1;
if (GetBoolean(features, SkipSha256Value))
omitFromScan &= ~Hash.SHA256; // TODO: This needs to be inverted later
includeInScan |= Hash.SHA256; // TODO: This needs to be inverted later
if (GetBoolean(features, SkipSha384Value))
omitFromScan &= ~Hash.SHA384; // TODO: This needs to be inverted later
includeInScan |= Hash.SHA384; // TODO: This needs to be inverted later
if (GetBoolean(features, SkipSha512Value))
omitFromScan &= ~Hash.SHA512; // TODO: This needs to be inverted later
includeInScan |= Hash.SHA512; // TODO: This needs to be inverted later
if (GetBoolean(features, SkipSpamSumValue))
omitFromScan &= ~Hash.SpamSum; // TODO: This needs to be inverted later
includeInScan |= Hash.SpamSum; // TODO: This needs to be inverted later
return omitFromScan;
return includeInScan;
}
/// <summary>

6
SabreTools/Features/DatFromDir.cs
View File

@@ -4,7 +4,6 @@ using System.IO;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Tools;
namespace SabreTools.Features
{
@@ -62,7 +61,7 @@ namespace SabreTools.Features
bool addFileDates = GetBoolean(features, AddDateValue);
TreatAsFile asFiles = GetTreatAsFiles(features);
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var omitFromScan = GetOmitFromScan(features);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
var splitType = GetSplitType(features);
@@ -70,7 +69,6 @@ namespace SabreTools.Features
if (Cleaner.ExcludeFields == null)
Cleaner.ExcludeFields = new List<Field>();
Cleaner.ExcludeFields.AddRange(omitFromScan.AsFields());
if (!addFileDates)
Cleaner.ExcludeFields.Add(Field.DatItem_Date);
@@ -96,7 +94,7 @@ namespace SabreTools.Features
asFiles,
skipFileType,
addBlankFiles,
quickScan: omitFromScan == Hash.SecureHashes);
hashes: includeInScan);
if (success)
{