claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ALL] Rename a few things; fix GetSingleStreamInfo; add headerless DFD

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-03 15:05:07 -07:00
parent c280253396
commit ccb063c74e
11 changed files with 189 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
SabreTools.Helper/Data/Build.cs
View File

@@ -134,6 +134,7 @@ namespace SabreTools.Helper
helptext.Add(" -ab, --add-blank Output blank files for folders");
helptext.Add(" -ad, --add-date Output dates for each file parsed");
helptext.Add(" -cf, --copy-files Copy files to the temp directory before parsing");
helptext.Add(" -rh, --rem-head Remove headers from hash calculations");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -mt={4} Amount of threads to use (-1 unlimted)");
helptext.Add(" -es, --ext-split Split a DAT by two file extensions");

24
SabreTools.Helper/Objects/Dat/DatFile.cs
View File

@@ -4564,10 +4564,11 @@ namespace SabreTools.Helper
/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
/// <param name="copyFiles">True if files should be copied to the temp directory before hashing, false otherwise</param>
/// <param name="removeHeader">True if headers should be removed from files if possible, false otherwise</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file output</param>
public bool PopulateDatFromDir(string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, bool removeHeader, int maxDegreeOfParallelism, Logger logger)
{
// If the description is defined but not the name, set the name from the description
if (String.IsNullOrEmpty(Name) && !String.IsNullOrEmpty(Description))
@@ -4598,7 +4599,7 @@ namespace SabreTools.Helper
item =>
{
DFDProcessPossibleArchive(item, basePath, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, addBlanks, addDate,
tempDir, copyFiles, maxDegreeOfParallelism, logger);
tempDir, copyFiles, removeHeader, maxDegreeOfParallelism, logger);
});
// Now find all folders that are empty, if we are supposed to
@@ -4687,10 +4688,11 @@ namespace SabreTools.Helper
/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
/// <param name="copyFiles">True if files should be copied to the temp directory before hashing, false otherwise</param>
/// <param name="removeHeader">True if headers should be removed from files if possible, false otherwise</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file output</param>
private void DFDProcessPossibleArchive(string item, string basePath, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles,
bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, int maxDegreeOfParallelism, Logger logger)
bool enableGzip, bool addBlanks, bool addDate, string tempDir, bool copyFiles, bool removeHeader, int maxDegreeOfParallelism, Logger logger)
{
// Define the temporary directory
string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar;
@@ -4760,7 +4762,7 @@ namespace SabreTools.Helper
// Otherwise, just get the info on the file itself
else if (File.Exists(newItem))
{
DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, removeHeader, logger);
}
}
// Otherwise, attempt to extract the files to the temporary directory
@@ -4792,13 +4794,14 @@ namespace SabreTools.Helper
noMD5,
noSHA1,
addDate,
removeHeader,
logger);
});
}
// Otherwise, just get the info on the file itself
else if (File.Exists(newItem))
{
DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, logger);
DFDProcessFile(newItem, "", newBasePath, noMD5, noSHA1, addDate, removeHeader, logger);
}
}
@@ -4823,12 +4826,17 @@ namespace SabreTools.Helper
/// Process a single file as a file
/// </summary>
/// <param name="item">File to be added</param>
/// <param name="basepath">Path the represents the parent directory</param>
/// <param name="parent">Parent game to be used</param>
private void DFDProcessFile(string item, string parent, string basePath, bool noMD5, bool noSHA1, bool addDate, Logger logger)
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="noMD5">True if MD5 hashes should be skipped over, false otherwise</param>
/// <param name="noSHA1">True if SHA-1 hashes should be skipped over, false otherwise</param>
/// <param name="addDate">True if dates should be archived for all files, false otherwise</param>
/// <param name="removeHeader">True if headers should be removed from files if possible, false otherwise</param>
/// <param name="logger">Logger object for console and file output</param>
private void DFDProcessFile(string item, string parent, string basePath, bool noMD5, bool noSHA1, bool addDate, bool removeHeader, Logger logger)
{
logger.Verbose(Path.GetFileName(item) + " treated like a file");
Rom rom = FileTools.GetSingleFileInfo(item, noMD5: noMD5, noSHA1: noSHA1, date: addDate);
Rom rom = FileTools.GetSingleFileInfo(item, logger, noMD5: noMD5, noSHA1: noSHA1, date: addDate, removeHeader: removeHeader);
DFDProcessFileHelper(item, rom, basePath, parent, logger);
}

24
SabreTools.Helper/Objects/SimpleSort.cs
View File

@@ -138,7 +138,8 @@ namespace SabreTools.Helper
foreach (string input in _inputs)
{
_datdata.PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */,
true /* enableGzip */, false /* addBlanks */, false /* addDate */, "" /* tempDir */, false /* copyFiles */, 4 /* maxDegreeOfParallelism */, _logger);
true /* enableGzip */, false /* addBlanks */, false /* addDate */, "" /* tempDir */, false /* copyFiles */,
false /* removeHeader */, 4 /* maxDegreeOfParallelism */, _logger);
}
// Setup the fixdat
@@ -354,7 +355,7 @@ namespace SabreTools.Helper
// Hash and match the external files
if (shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat);
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// If we should scan the file as an archive
@@ -370,7 +371,7 @@ namespace SabreTools.Helper
// Now add all of the roms to the DAT
for (int i = 0; i < internalRomData.Count; i++)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat);
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
}
// Otherwise, try to extract the file to the temp folder
@@ -385,13 +386,13 @@ namespace SabreTools.Helper
List<string> extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
foreach (string extractedFile in extractedFiles)
{
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat);
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
}
}
// Otherwise, skip extracting and just get information on the file itself (if we didn't already)
else if (!shouldExternalScan)
{
RebuildToOutputAlternateParseRomHelper(file, ref matchdat);
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
}
// Clean the temp directory for the next round
@@ -455,10 +456,11 @@ namespace SabreTools.Helper
/// </summary>
/// <param name="file">Name of the file to attempt to add</param>
/// <param name="matchdat">Reference to the Dat to add to</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if the file could be added, false otherwise</returns>
public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat)
public bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
{
Rom rom = FileTools.GetSingleFileInfo(file);
Rom rom = FileTools.GetSingleFileInfo(file, logger);
// If we have a blank RomData, it's an error
if (rom.Name == null)
@@ -494,7 +496,7 @@ namespace SabreTools.Helper
// Transform the stream and get the information from it
Skippers.TransformStream(input, output, rule, _logger, false, true);
Rom romNH = FileTools.GetSingleStreamInfo(output);
Rom romNH = FileTools.GetSingleStreamInfo(output, output.Length);
romNH.Name = "HEAD::" + rom.Name;
romNH.MachineName = rom.MachineName;
@@ -544,7 +546,7 @@ namespace SabreTools.Helper
// Hash and match the external files
if (shouldExternalScan)
{
Rom rom = FileTools.GetSingleFileInfo(input);
Rom rom = FileTools.GetSingleFileInfo(input, _logger);
// If we have a blank RomData, it's an error
if (rom.Name == null)
@@ -614,7 +616,7 @@ namespace SabreTools.Helper
// Otherwise, apply the rule to the file
string newinput = input + ".new";
Skippers.TransformFile(input, newinput, rule, _logger);
Rom drom = FileTools.GetSingleFileInfo(newinput);
Rom drom = FileTools.GetSingleFileInfo(newinput, _logger);
// If we have a blank RomData, it's an error
if (drom.Name == null)
@@ -912,7 +914,7 @@ namespace SabreTools.Helper
ArchiveTools.ExtractArchive(Path.GetFullPath(archive), temparcdir, _logger);
foreach (string tempfile in Directory.EnumerateFiles(temparcdir, "*", SearchOption.AllDirectories))
{
roms.Add(FileTools.GetSingleFileInfo(Path.GetFullPath(tempfile)));
roms.Add(FileTools.GetSingleFileInfo(Path.GetFullPath(tempfile), _logger));
}
// Clear the temporary archive directory

17
SabreTools.Helper/README.1ST
View File

@@ -154,10 +154,9 @@ Options:
Normally, the DAT will be created with the date in the file name. This flag removes
that but keeps the date tag intact.
-u, --unzip Force unzipping in created DAT
This sets the 'forcepacking="unzip"' flag in the outputted DAT. When used with a
file manager that allows for it, this will force the outputted files to be in
subdirectories instead of archives.
-fp=, --forcepack= Set force packing
Set the forcepacking flag to one of the supported values:
None, Zip, Unzip
-f, --files Treat archives as files
Instead of trying to enumerate the files within archives, treat the archives as
@@ -241,6 +240,11 @@ Options:
If this flag is set, then all files that are going to be parsed are moved to the temporary directory before being hashed. This can be helpful in cases where the
temp folder is located on an SSD and the user wants to take advantage of this.
-rh, --rem-head Remove headers from hash calculations
If this flag is set, then all files that have copier headers that are detected will
have them removed from the hash calculation. This will allow for a headered collection
to be hashed without possibly variant information.
-t=, --temp= Set the name of the temporary directory
Optionally, a temp folder can be supplied in the case the default temp directory
(inside the running folder) is not preferred. This is used for any operations
@@ -1101,6 +1105,11 @@ Below are originally from SabreTools / DATabase -
By default, this will set the forcepacking="unzip" flag on the output DAT. This
flag disables this output
-u, --unzip Force unzipping in created DAT
This sets the 'forcepacking="unzip"' flag in the outputted DAT. When used with a
file manager that allows for it, this will force the outputted files to be in
subdirectories instead of archives.
Below are originally from SingleGame (Standalone) -
-r=rootdir Set the directory name for path size

5
SabreTools.Helper/Skippers/Skippers.cs
View File

@@ -324,7 +324,10 @@ namespace SabreTools.Helper
// Loop through and find a Skipper that has the right name
logger.Verbose("Beginning search for matching header skip rules");
foreach (Skipper skipper in List)
List<Skipper> tempList = new List<Skipper>();
tempList.AddRange(List);
foreach (Skipper skipper in tempList)
{
if (String.IsNullOrEmpty(skipperName) || (!String.IsNullOrEmpty(skipper.Name) && skipperName.ToLowerInvariant() == skipper.Name.ToLowerInvariant()))
{

2
SabreTools.Helper/Tools/ArchiveTools.cs
View File

@@ -965,7 +965,7 @@ namespace SabreTools.Helper
outDir = Path.GetFullPath(outDir);
// Now get the Rom info for the file so we have hashes and size
Rom rom = FileTools.GetSingleFileInfo(input);
Rom rom = FileTools.GetSingleFileInfo(input, logger);
// If it doesn't exist, create the output file and then write
string outfile = Path.Combine(outDir, rom.SHA1 + ".gz");

49
SabreTools.Helper/Tools/FileTools.cs
View File

@@ -151,12 +151,14 @@ namespace SabreTools.Helper
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="logger">Logger object for console and file output</param>
/// <param name="noMD5">True if MD5 hashes should not be calculated, false otherwise (default)</param>
/// <param name="noSHA1">True if SHA-1 hashes should not be calcluated, false otherwise (default)</param>
/// <param name="offset">Set a >0 number for getting hash for part of the file, 0 otherwise (default)</param>
/// <param name="date">True if the file Date should be included, false otherwise (default)</param>
/// <param name="removeHeader">True if headers should be removed from files if possible, false otherwise (default)</param>
/// <returns>Populated RomData object if success, empty one on error</returns>
public static Rom GetSingleFileInfo(string input, bool noMD5 = false, bool noSHA1 = false, long offset = 0, bool date = false)
public static Rom GetSingleFileInfo(string input, Logger logger, bool noMD5 = false, bool noSHA1 = false, long offset = 0, bool date = false, bool removeHeader = false)
{
// Add safeguard if file doesn't exist
if (!File.Exists(input))
@@ -165,7 +167,36 @@ namespace SabreTools.Helper
}
// Get the information from the file stream
Rom rom = GetSingleStreamInfo(File.OpenRead(input), noMD5, noSHA1, offset, false);
Rom rom = new Rom();
if (removeHeader)
{
SkipperRule rule = Skippers.MatchesSkipper(input, "", logger);
// If there's a match, get the new information from the stream
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the input and output streams
MemoryStream outputStream = new MemoryStream();
FileStream inputStream = File.OpenRead(input);
// Transform the stream and get the information from it
Skippers.TransformStream(inputStream, outputStream, rule, logger, false, true);
rom = GetSingleStreamInfo(outputStream, outputStream.Length);
// Dispose of the streams
outputStream.Dispose();
inputStream.Dispose();
}
// Otherwise, just get the info
else
{
rom = GetSingleStreamInfo(File.OpenRead(input), new FileInfo(input).Length, noMD5, noSHA1, offset, false);
}
}
else
{
rom = GetSingleStreamInfo(File.OpenRead(input), new FileInfo(input).Length, noMD5, noSHA1, offset, false);
}
// Add unique data from the file
rom.Name = Path.GetFileName(input);
@@ -330,7 +361,7 @@ namespace SabreTools.Helper
}
// Now add the information to the database if it's not already there
Rom rom = GetSingleFileInfo(newfile);
Rom rom = GetSingleFileInfo(newfile, logger);
DatabaseTools.AddHeaderToDatabase(hstr, rom.SHA1, rule.SourceFile, logger);
return true;
@@ -354,7 +385,7 @@ namespace SabreTools.Helper
bool success = true;
// First, get the SHA-1 hash of the file
Rom rom = GetSingleFileInfo(file);
Rom rom = GetSingleFileInfo(file, logger);
// Then try to pull the corresponding headers from the database
string header = "";
@@ -428,12 +459,13 @@ namespace SabreTools.Helper
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="size">Size of the input stream</param>
/// <param name="noMD5">True if MD5 hashes should not be calculated, false otherwise (default)</param>
/// <param name="noSHA1">True if SHA-1 hashes should not be calcluated, false otherwise (default)</param>
/// <param name="offset">Set a >0 number for getting hash for part of the file, 0 otherwise (default)</param>
/// <param name="keepReadOpen">True if the underlying read stream should be kept open, false otherwise</param>
/// <returns>Populated RomData object if success, empty one on error</returns>
public static Rom GetSingleStreamInfo(Stream input, bool noMD5 = false, bool noSHA1 = false, long offset = 0, bool keepReadOpen = false)
public static Rom GetSingleStreamInfo(Stream input, long size, bool noMD5 = false, bool noSHA1 = false, long offset = 0, bool keepReadOpen = false)
{
// If we have a negative offset, zero it out since we don't support it yet
if (offset < 0)
@@ -444,7 +476,7 @@ namespace SabreTools.Helper
Rom rom = new Rom
{
Type = ItemType.Rom,
Size = input.Length - Math.Abs(offset),
Size = size,
CRC = string.Empty,
MD5 = string.Empty,
SHA1 = string.Empty,
@@ -458,10 +490,7 @@ namespace SabreTools.Helper
SHA1 sha1 = SHA1.Create();
// Seek to the starting position, if one is set
if (offset > 0)
{
input.Seek(offset, SeekOrigin.Begin);
}
input.Seek(offset, SeekOrigin.Begin);
byte[] buffer = new byte[1024];
int read;