[ALL] Add documentation for SimpleSort in SabreTools; port over Verify

This commit is contained in:
Matt Nadareski
2016-10-20 17:24:44 -07:00
parent 64c95b05c9
commit e67b85e901
8 changed files with 556 additions and 288 deletions

View File

@@ -184,8 +184,8 @@ namespace SabreTools
// Create the sorting object to use and rebuild the needed files // Create the sorting object to use and rebuild the needed files
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1));
SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, false, true, true, asl, false, null, _logger); SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, _logger);
ss.StartProcessing(); ss.RebuildToOutput();
} }
/// <summary> /// <summary>
@@ -223,8 +223,8 @@ namespace SabreTools
// Now scan all of those depots and rebuild // Now scan all of those depots and rebuild
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, false, copy, copy, asl, false, null, _logger); SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, _logger);
ss.StartProcessing(); ss.RebuildToOutput();
} }
} }

View File

@@ -108,6 +108,21 @@ namespace SabreTools.Helper
helptext.Add(""); helptext.Add("");
helptext.Add("Options:"); helptext.Add("Options:");
helptext.Add(" -?, -h, --help Show this help"); helptext.Add(" -?, -h, --help Show this help");
// Convert
helptext.Add(" -cv, --convert Enable conversion of input files to TZip");
helptext.Add(" -dat= DAT to be used as a filter for conversion");
helptext.Add(" -out= Output directory");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -del, --delete Delete input files [DO NOT USE]");
helptext.Add(" -tgz Enable TorrentGZ output");
helptext.Add(" -r, --romba Enable Romba depot dir output");
helptext.Add(" -7z={0} Set scanning level for 7z archives");
helptext.Add(" -gz={2} Set scanning level for GZip archives");
helptext.Add(" -rar={2} Set scanning level for RAR archives");
helptext.Add(" -zip={0} Set scanning level for ZIP archives");
// DATFromDir
helptext.Add(" -d, --dfd Create a DAT from an input directory"); helptext.Add(" -d, --dfd Create a DAT from an input directory");
helptext.Add(" -nm, --noMD5 Don't include MD5 in output"); helptext.Add(" -nm, --noMD5 Don't include MD5 in output");
helptext.Add(" -ns, --noSHA1 Don't include SHA1 in output"); helptext.Add(" -ns, --noSHA1 Don't include SHA1 in output");
@@ -144,15 +159,41 @@ namespace SabreTools.Helper
helptext.Add(" -h=, --header= Set a header skipper to use, blank means all"); helptext.Add(" -h=, --header= Set a header skipper to use, blank means all");
helptext.Add(" -t=, --temp= Set the temporary directory to use"); helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -mt={4} Amount of threads to use (-1 unlimted)"); helptext.Add(" -mt={4} Amount of threads to use (-1 unlimted)");
// Extension Split
helptext.Add(" -es, --ext-split Split a DAT by two file extensions"); helptext.Add(" -es, --ext-split Split a DAT by two file extensions");
helptext.Add(" -exta= First set of extensions (comma-separated)"); helptext.Add(" -exta= First set of extensions (comma-separated)");
helptext.Add(" -extb= Second set of extensions (comma-separated)"); helptext.Add(" -extb= Second set of extensions (comma-separated)");
helptext.Add(" -out= Output directory"); helptext.Add(" -out= Output directory");
// Headerer
helptext.Add(" -he, --headerer Extract and remove copier headers"); helptext.Add(" -he, --headerer Extract and remove copier headers");
helptext.Add(" -r, --restore Restore header to file based on SHA-1 instead"); helptext.Add(" -r, --restore Restore header to file based on SHA-1 instead");
helptext.Add(" -out= Output directory"); helptext.Add(" -out= Output directory");
// Hash Split
helptext.Add(" -hs, --hash-split Split a DAT or folder by best-available hashes"); helptext.Add(" -hs, --hash-split Split a DAT or folder by best-available hashes");
helptext.Add(" -out= Output directory"); helptext.Add(" -out= Output directory");
// Sort
helptext.Add(" -ss, --sort Sort input files by a set of DATs");
helptext.Add(" -dat= Input DAT to rebuild against");
helptext.Add(" -out= Output directory");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -del, --delete Delete input files [DO NOT USE]");
helptext.Add(" -qs, --quick Enable quick scanning of archives");
helptext.Add(" -ad, --add-date Add original dates from DAT, if possible");
helptext.Add(" -tgz Enable TorrentGZ output");
helptext.Add(" -r, --romba Enable Romba depot dir output");
helptext.Add(" -do, --directory Output files as uncompressed");
helptext.Add(" -h=, --header= Set a header skipper to use, blank means all");
helptext.Add(" -7z={0} Set scanning level for 7z archives");
helptext.Add(" -gz={2} Set scanning level for GZip archives");
helptext.Add(" -rar={2} Set scanning level for RAR archives");
helptext.Add(" -zip={0} Set scanning level for ZIP archives");
helptext.Add(" -ud, --update-dat Output updated DAT");
// Stats
helptext.Add(" -st, --stats Get statistics on all input DATs"); helptext.Add(" -st, --stats Get statistics on all input DATs");
helptext.Add(" -bc, --baddump-col Add baddump stats to output"); helptext.Add(" -bc, --baddump-col Add baddump stats to output");
helptext.Add(" -csv, --csv Output in Comma-Separated Value format"); helptext.Add(" -csv, --csv Output in Comma-Separated Value format");
@@ -161,8 +202,12 @@ namespace SabreTools.Helper
helptext.Add(" -nc, --nodump-col Add nodump stats to output"); helptext.Add(" -nc, --nodump-col Add nodump stats to output");
helptext.Add(" -si, --single Show individual statistics"); helptext.Add(" -si, --single Show individual statistics");
helptext.Add(" -tsv, --tsv Output in Tab-Separated Value format"); helptext.Add(" -tsv, --tsv Output in Tab-Separated Value format");
// Type Split
helptext.Add(" -ts, --type-split Split a DAT or folder by file types (rom/disk)"); helptext.Add(" -ts, --type-split Split a DAT or folder by file types (rom/disk)");
helptext.Add(" -out= Output directory"); helptext.Add(" -out= Output directory");
// Update
helptext.Add(" -ud, --update Update a DAT file"); helptext.Add(" -ud, --update Update a DAT file");
helptext.Add(" -oa, --output-all Output in all formats"); helptext.Add(" -oa, --output-all Output in all formats");
helptext.Add(" -oc, --output-cmp Output in CMP format"); helptext.Add(" -oc, --output-cmp Output in CMP format");
@@ -260,9 +305,18 @@ namespace SabreTools.Helper
helptext.Add(" Supported values are:"); helptext.Add(" Supported values are:");
helptext.Add(" None, Good, BadDump, Nodump, Verified, NotNodump"); helptext.Add(" None, Good, BadDump, Nodump, Verified, NotNodump");
helptext.Add(" -out= Output directory (overridden by --inplace)"); helptext.Add(" -out= Output directory (overridden by --inplace)");
// Verify
helptext.Add(" -ve, --verify Verify a folder against DATs");
helptext.Add(" -dat= Input DAT to verify against");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -h=, --header= Set a header skipper to use, blank means all");
// Additional Notes
helptext.Add(""); helptext.Add("");
helptext.Add("Filenames and directories can't start with a reserved string"); helptext.Add("Filenames and directories can't start with a reserved string");
helptext.Add("unless prefixed by 'input='"); helptext.Add("unless prefixed by 'input='");
helptext.Add(""); helptext.Add("");
helptext.Add("Filter parameters game name, rom name, CRC, MD5, SHA-1 can"); helptext.Add("Filter parameters game name, rom name, CRC, MD5, SHA-1 can");
helptext.Add("do partial matches using asterisks as follows (case insensitive):"); helptext.Add("do partial matches using asterisks as follows (case insensitive):");
@@ -270,6 +324,7 @@ namespace SabreTools.Helper
helptext.Add(" 00* means starts with '00'"); helptext.Add(" 00* means starts with '00'");
helptext.Add(" *00* means contains '00'"); helptext.Add(" *00* means contains '00'");
helptext.Add(" 00 means exactly equals '00'"); helptext.Add(" 00 means exactly equals '00'");
helptext.Add(""); helptext.Add("");
helptext.Add("Filter parameters for size can use postfixes for inputs:"); helptext.Add("Filter parameters for size can use postfixes for inputs:");
helptext.Add(" e.g. 8kb => 8000 or 8kib => 8192"); helptext.Add(" e.g. 8kb => 8000 or 8kib => 8192");
@@ -281,6 +336,8 @@ namespace SabreTools.Helper
helptext.Add(""); helptext.Add("");
helptext.Add("Options:"); helptext.Add("Options:");
helptext.Add(" -?, -h, --help Show this help"); helptext.Add(" -?, -h, --help Show this help");
// Convert, Sort, Verify
helptext.Add(" -dat= Input DAT to rebuild against (REQUIRED)"); helptext.Add(" -dat= Input DAT to rebuild against (REQUIRED)");
helptext.Add(" -out= Output directory"); helptext.Add(" -out= Output directory");
helptext.Add(" -t=, --temp= Set the temporary directory to use"); helptext.Add(" -t=, --temp= Set the temporary directory to use");
@@ -299,6 +356,8 @@ namespace SabreTools.Helper
helptext.Add(" -rar={2} Set scanning level for RAR archives"); helptext.Add(" -rar={2} Set scanning level for RAR archives");
helptext.Add(" -zip={0} Set scanning level for ZIP archives"); helptext.Add(" -zip={0} Set scanning level for ZIP archives");
helptext.Add(" -ud, --update-dat Output updated DAT"); helptext.Add(" -ud, --update-dat Output updated DAT");
// Additional Notes
helptext.Add(""); helptext.Add("");
helptext.Add("Archive scanning levels:"); helptext.Add("Archive scanning levels:");
helptext.Add(" 0 Hash archive and contents"); helptext.Add(" 0 Hash archive and contents");

View File

@@ -16,7 +16,6 @@ namespace SabreTools.Helper
private bool _quickScan; private bool _quickScan;
private bool _date; private bool _date;
private bool _toFolder; private bool _toFolder;
private bool _verify;
private bool _delete; private bool _delete;
private bool _tgz; private bool _tgz;
private bool _romba; private bool _romba;
@@ -41,7 +40,6 @@ namespace SabreTools.Helper
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="toFolder">True if files should be output to folder, false otherwise</param> /// <param name="toFolder">True if files should be output to folder, false otherwise</param>
/// <param name="verify">True if output directory should be checked instead of rebuilt to, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param> /// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param> /// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
@@ -50,7 +48,7 @@ namespace SabreTools.Helper
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
public SimpleSort(DatFile datdata, List<string> inputs, string outDir, string tempDir, public SimpleSort(DatFile datdata, List<string> inputs, string outDir, string tempDir,
bool quickScan, bool date, bool toFolder, bool verify, bool delete, bool tgz, bool romba, bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba,
ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, Logger logger) ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, Logger logger)
{ {
_datdata = datdata; _datdata = datdata;
@@ -60,7 +58,6 @@ namespace SabreTools.Helper
_quickScan = quickScan; _quickScan = quickScan;
_date = date; _date = date;
_toFolder = toFolder; _toFolder = toFolder;
_verify = verify;
_delete = delete; _delete = delete;
_tgz = tgz; _tgz = tgz;
_romba = romba; _romba = romba;
@@ -78,10 +75,21 @@ namespace SabreTools.Helper
} }
/// <summary> /// <summary>
/// Pick the appropriate action based on the inputs /// Process the DAT and find all matches in input files and folders
/// </summary> /// </summary>
/// <returns>True if success, false otherwise</returns> /// <returns>True if rebuilding was a success, false otherwise</returns>
public bool StartProcessing() /// <remarks>
/// This currently processes files as follows:
/// 1) Get all file names from the input files/folders
/// 2) Loop through and process each file individually
/// a) Hash the file
/// b) Check against the DAT for duplicates
/// c) Check for headers
/// d) Check headerless rom for duplicates
///
/// This is actually rather slow and inefficient. See below for more correct implemenation
/// </remarks>
public bool RebuildToOutput()
{ {
// First, check that the output directory exists // First, check that the output directory exists
if (!Directory.Exists(_outDir)) if (!Directory.Exists(_outDir))
@@ -100,108 +108,6 @@ namespace SabreTools.Helper
FileTools.CleanDirectory(_tempDir); FileTools.CleanDirectory(_tempDir);
} }
if (_verify)
{
return VerifyDirectory();
}
else
{
return RebuildToOutput();
}
}
/// <summary>
/// Process the DAT and verify the output directory
/// </summary>
/// <returns>True if verification was a success, false otherwise</returns>
private bool VerifyDirectory()
{
bool success = true;
// Enumerate all files from the output directory
List<string> files = new List<string>();
foreach (string file in Directory.EnumerateFiles(_outDir, "*", SearchOption.AllDirectories))
{
_logger.Verbose("File found: '" + file + "'");
files.Add(Path.GetFullPath(file));
}
/*
We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder
*/
// Then, loop through and check each of the inputs
_logger.User("Processing files:\n");
foreach (string input in _inputs)
{
_datdata.PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */,
true /* enableGzip */, false /* addBlanks */, false /* addDate */, "" /* tempDir */, false /* copyFiles */,
_headerToCheckAgainst, 4 /* maxDegreeOfParallelism */, _logger);
}
// Setup the fixdat
_matched = (DatFile)_datdata.CloneHeader();
_matched.Files = new SortedDictionary<string, List<DatItem>>();
_matched.FileName = "fixDat_" + _matched.FileName;
_matched.Name = "fixDat_" + _matched.Name;
_matched.Description = "fixDat_" + _matched.Description;
_matched.OutputFormat = OutputFormat.Logiqx;
// Now that all files are parsed, get only files found in directory
bool found = false;
foreach (List<DatItem> roms in _datdata.Files.Values)
{
List<DatItem> newroms = DatItem.Merge(roms, _logger);
foreach (Rom rom in newroms)
{
if (rom.SourceID == 99)
{
found = true;
string key = rom.Size + "-" + rom.CRC;
if (_matched.Files.ContainsKey(key))
{
_matched.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
_matched.Files.Add(key, temp);
}
}
}
}
// Now output the fixdat to the main folder
if (found)
{
_matched.WriteToFile("", _logger, stats: true);
}
else
{
_logger.User("No fixDat needed");
}
return success;
}
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <returns>True if rebuilding was a success, false otherwise</returns>
/// <remarks>
/// This currently processes files as follows:
/// 1) Get all file names from the input files/folders
/// 2) Loop through and process each file individually
/// a) Hash the file
/// b) Check against the DAT for duplicates
/// c) Check for headers
/// d) Check headerless rom for duplicates
///
/// This is actually rather slow and inefficient. See below for more correct implemenation
/// </remarks>
private bool RebuildToOutput()
{
bool success = true; bool success = true;
_logger.User("Retrieving list all files from input"); _logger.User("Retrieving list all files from input");
@@ -636,8 +542,25 @@ namespace SabreTools.Helper
/// 4) Order by output game /// 4) Order by output game
/// 5) Rebuild all files /// 5) Rebuild all files
/// </remarks> /// </remarks>
private bool RebuiltToOutputAlternate() public bool RebuiltToOutputAlternate()
{ {
// First, check that the output directory exists
if (!Directory.Exists(_outDir))
{
Directory.CreateDirectory(_outDir);
_outDir = Path.GetFullPath(_outDir);
}
// Then create or clean the temp directory
if (!Directory.Exists(_tempDir))
{
Directory.CreateDirectory(_tempDir);
}
else
{
FileTools.CleanDirectory(_tempDir);
}
bool success = true; bool success = true;
#region Find all files #region Find all files

View File

@@ -140,6 +140,54 @@ Options:
-?, -h, --help Show the built-in help text -?, -h, --help Show the built-in help text
Built-in to most of the programs is a basic help text Built-in to most of the programs is a basic help text
-cv, --convert Enable conversion of input files to TorrentZip
Using a folder or set of folders, convert the input archives to TorrentZip, rebuilding
to another folder. This is useful for rebuilding non-TorrentZipped sets without having
to run them through a larger tool or helping to rebuild a romba depot
-dat= Name of the DAT to be used as a filter
A supplied DAT file to be used as a filter in conversion. If a file is found in the
DAT, it will be skipped on output. This allows convert to act like an anti-sort,
useful for finding useless files in an input folder.
-out= Set the name of the output directory
This sets an output folder to be used when the files are created. If a path
is not defined, the application directory is used instead.
-t=, --temp= Set the name of the temporary directory
Optionally, a temp folder can be supplied in the case the default temp directory
(inside the running folder) is not preferred. This is used for any operations
that require an archive to be extracted.
-del, --delete Delete input files
This is a WIP flag that allows for deletion of input files once they have been
rebuilt. It is not recommended for normal use because it does not discriminate
whether or not the input files were rebuilt or not before deletion
-tgz Enable Torrent GZ output
Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ
(TGZ) files. This format is based on the GZip archive format, but with custom header
information and a file name replaced by the SHA-1 of the file inside. This is
primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but
may be used more widely in the future.
-r, --romba Enable Romba depot directory output
As an extension of the parent flag, this outputs the TGZ files into directories
based on the structure used by Romba. This uses nested folders using the first
4 bytes of the SHA-1, 1 byte for each layer of the directory name. It also
includes two auxilary files, .romba_size and .romba_size.backup, that have the
compressed size of the folder inside for use with Romba.
-7z={0} Set scanning level for 7z archives
-gz={2} Set scanning level for GZip archives
-rar={2} Set scanning level for RAR archives
-zip={0} Set scanning level for ZIP archives
For each of the major archive types recognized by the libraries used by this
program, scan the archive in one of the following ways:
0 Hash both archive and its contents
1 Only hash contents of the archive
2 Only hash archive itself (treat like a regular file)
-d, --dfd Create a DAT from each input directory -d, --dfd Create a DAT from each input directory
Create a DAT file from an input directory or set of files. By default, this will Create a DAT file from an input directory or set of files. By default, this will
output a DAT named based on the input directory and the current date. It will also output a DAT named based on the input directory and the current date. It will also
@@ -209,7 +257,7 @@ Options:
-ox, -output-xml Output in Logiqx XML format (default) -ox, -output-xml Output in Logiqx XML format (default)
Add outputting the created DAT to Logiqx XML format Add outputting the created DAT to Logiqx XML format
-gz, --gz-files Allow reading of GZIP files as archives -gzf, --gz-files Allow reading of GZIP files as archives
Since GZip files are not commonly used for file storage, this flag allows for Since GZip files are not commonly used for file storage, this flag allows for
any GZip archives to have their contents hashed instead. any GZip archives to have their contents hashed instead.
@@ -339,6 +387,82 @@ Options:
This should only be used if one of the inputs starts with a flag or another already This should only be used if one of the inputs starts with a flag or another already
defined input. defined input.
-ss, --sort Sort input files by a set of DATs
This feature allows the user to quickly rebuild based on a supplied DAT file(s). By
default all files will be rebuilt to TorrentZip (TZ) files. This format is based on
the ZIP archive format, but with custom header information. This is primarily used by
external tool RomVault (http://www.romvault.com/) and is already widely used.
-dat= Name of the DAT to be used for the various options
The user-supplied DAT used to check which files need to be rebuilt. Multiple
occurrences of this flag are allowed.
-out= Set the name of the output directory
This sets an output folder to be used when the files are created. If a path
is not defined, the application directory is used instead.
-t=, --temp= Set the name of the temporary directory
Optionally, a temp folder can be supplied in the case the default temp directory
(inside the running folder) is not preferred. This is used for any operations that
require an archive to be extracted.
-d, --delete Enable deletion of the input files
Optionally, the input files, once processed, can be deleted. This can be useful
when the original file structure is no longer needed or if there is limited space
on the source drive.
-qs, --quick Enable quick scanning of archives
For all archives, if this flag is enabled, it will only use the header information
to get the archive entries' file information. The upside to this is that it is much
quicker than extracting all files to the temp folder. On the downside, it can only
get the CRC and size from most archive formats, leading to possible issues.
-ad, --add-date Write dates for each file parsed, if available
If this flag is set, the the date in the DAT will be used for the output file
instead of the standard date and time for TorrentZip. This will technically
invalidate the output files as proper TorrentZip files because the date will not
match the standard.
-tgz Enable Torrent GZ output
Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ
(TGZ) files. This format is based on the GZip archive format, but with custom header
information and a file name replaced by the SHA-1 of the file inside. This is
primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but
may be used more widely in the future.
-r, --romba Enable Romba depot directory output
As an extension of the parent flag, this outputs the TGZ files into directories
based on the structure used by Romba. This uses nested folders using the first
4 bytes of the SHA-1, 1 byte for each layer of the directory name. It also
includes two auxilary files, .romba_size and .romba_size.backup, that have the
compressed size of the folder inside for use with Romba.
-do, --directory Enable outputting files uncompressed
Instead of outputting the files to ZIP archives, files will be rebuilt to named
subdirectories within the output folder. This is useful for when the DAT does not
already have the flag specified.
-h=, --header= Remove headers from hash calculations
If this is set, then all files that have copier headers that are detected will
have them removed from the hash calculation. This will allow for a headered collection
to be hashed without possibly variant information. If a particular header skipper is
defined, and that skipper exists, then it will be used instead of trying to find one
that matches.
-7z={0} Set scanning level for 7z archives
-gz={2} Set scanning level for GZip archives
-rar={2} Set scanning level for RAR archives
-zip={0} Set scanning level for ZIP archives
For each of the major archive types recognized by the libraries used by this
program, scan the archive in one of the following ways:
0 Hash both archive and its contents
1 Only hash contents of the archive
2 Only hash archive itself (treat like a regular file)
-ud, --update-dat Output updated DAT (rebuild only)
Once the files that were able to rebuilt are taken care of, a DAT of the files
that could not be matched will be output to the program directory.
-st, --stats Get statistics on all input DATs -st, --stats Get statistics on all input DATs
This will output by default the combined statistics for all input DAT files. The stats This will output by default the combined statistics for all input DAT files. The stats
that are outputted are as follows: that are outputted are as follows:
@@ -669,6 +793,32 @@ Options:
This sets an output folder to be used when the files are created. If a path This sets an output folder to be used when the files are created. If a path
is not defined, the application directory is used instead. is not defined, the application directory is used instead.
-ve, --verify Verify a folder against an input DAT
When used, this will use an input DAT or set of DATs to blindly check against an input
folder. The base of the folder is considered the base for the combined DATs and games are
either the directories or archives within. This will only do a direct verification of the
items within and will create a fixdat afterwards for missing files.
-dat= Name of the DAT to be used for the various options
The user-supplied DAT used to check which files need to be verified. Multiple
occurrences of this flag are allowed.
-out= Set the name of the output directory
This sets an output folder to be used where the files will be verified. If a path
is not defined, the application directory is used instead.
-t=, --temp= Set the name of the temporary directory
Optionally, a temp folder can be supplied in the case the default temp directory
(inside the running folder) is not preferred. This is used for any operations that
require an archive to be extracted.
-h=, --header= Remove headers from hash calculations
If this is set, then all files that have copier headers that are detected will
have them removed from the hash calculation. This will allow for a headered collection
to be hashed without possibly variant information. If a particular header skipper is
defined, and that skipper exists, then it will be used instead of trying to find one
that matches.
** Section 2.3 - SimpleSort ** Section 2.3 - SimpleSort
SimpleSort is a WIP program that is meant as a command-line tool to quickly rebuild and SimpleSort is a WIP program that is meant as a command-line tool to quickly rebuild and
@@ -1006,7 +1156,7 @@ Below are originally from SabreTools / DATabase -
-ox, -output-xml Output in Logiqx XML format (default) -ox, -output-xml Output in Logiqx XML format (default)
Add outputting the created DAT to Logiqx XML format Add outputting the created DAT to Logiqx XML format
-gzf, -gz-files Allow reading of GZIP files as archives -gz, -gz-files Allow reading of GZIP files as archives
Since GZip files are not commonly used for file storage, this flag allows for Since GZip files are not commonly used for file storage, this flag allows for
any GZip archives to have their contents hashed instead. any GZip archives to have their contents hashed instead.

View File

@@ -454,6 +454,92 @@ namespace SabreTools.Helper
#endregion #endregion
#region Rebuilding and Verifying
/// <summary>
/// Process the DAT and verify the output directory
/// </summary>
/// <param name="datFile">DAT to use to verify the directory</param>
/// <param name="inputs">List of input directories to compare against</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if verification was a success, false otherwise</returns>
public static bool VerifyDirectory(DatFile datFile, List<string> inputs, string tempDir, string headerToCheckAgainst, Logger logger)
{
// First create or clean the temp directory
if (!Directory.Exists(tempDir))
{
Directory.CreateDirectory(tempDir);
}
else
{
CleanDirectory(tempDir);
}
bool success = true;
/*
We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder
*/
// Then, loop through and check each of the inputs
logger.User("Processing files:\n");
foreach (string input in inputs)
{
datFile.PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */,
true /* enableGzip */, false /* addBlanks */, false /* addDate */, tempDir /* tempDir */, false /* copyFiles */,
headerToCheckAgainst, 4 /* maxDegreeOfParallelism */, logger);
}
// Setup the fixdat
DatFile matched = (DatFile)datFile.CloneHeader();
matched.Files = new SortedDictionary<string, List<DatItem>>();
matched.FileName = "fixDat_" + matched.FileName;
matched.Name = "fixDat_" + matched.Name;
matched.Description = "fixDat_" + matched.Description;
matched.OutputFormat = OutputFormat.Logiqx;
// Now that all files are parsed, get only files found in directory
bool found = false;
foreach (List<DatItem> roms in datFile.Files.Values)
{
List<DatItem> newroms = DatItem.Merge(roms, logger);
foreach (Rom rom in newroms)
{
if (rom.SourceID == 99)
{
found = true;
string key = rom.Size + "-" + rom.CRC;
if (matched.Files.ContainsKey(key))
{
matched.Files[key].Add(rom);
}
else
{
List<DatItem> temp = new List<DatItem>();
temp.Add(rom);
matched.Files.Add(key, temp);
}
}
}
}
// Now output the fixdat to the main folder
if (found)
{
matched.WriteToFile("", logger, stats: true);
}
else
{
logger.User("No fixDat needed");
}
return success;
}
#endregion
#region Stream Information #region Stream Information
/// <summary> /// <summary>

View File

@@ -62,7 +62,7 @@ namespace SabreTools
_logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); _logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false,
false, false, delete, tgz, romba, asl, false, null, _logger); false, delete, tgz, romba, asl, false, null, _logger);
return ss.Convert(); return ss.Convert();
} }
@@ -300,7 +300,6 @@ namespace SabreTools
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="sevenzip">Integer representing the archive handling level for 7z</param> /// <param name="sevenzip">Integer representing the archive handling level for 7z</param>
/// <param name="toFolder">True if files should be output to folder, false otherwise</param> /// <param name="toFolder">True if files should be output to folder, false otherwise</param>
/// <param name="verify">True if output directory should be checked instead of rebuilt to, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="tgz">True to output files in TorrentGZ format, false for TorrentZip</param> /// <param name="tgz">True to output files in TorrentGZ format, false for TorrentZip</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param> /// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
@@ -309,8 +308,8 @@ namespace SabreTools
/// <param name="zip">Integer representing the archive handling level for Zip</param> /// <param name="zip">Integer representing the archive handling level for Zip</param>
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param> /// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
private static void InitSortVerify(List<string> datfiles, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date, private static void InitSort(List<string> datfiles, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
bool toFolder, bool verify, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst) bool toFolder, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst)
{ {
// Get the archive scanning level // Get the archive scanning level
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip);
@@ -327,8 +326,8 @@ namespace SabreTools
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date,
toFolder, verify, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, _logger); toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, _logger);
ss.StartProcessing(); ss.RebuildToOutput();
} }
/// <summary> /// <summary>
@@ -650,6 +649,32 @@ namespace SabreTools
gamename, romname, romtype, sgt, slt, seq, crc, md5, sha1, itemStatus, trim, single, root, maxDegreeOfParallelism, _logger); gamename, romname, romtype, sgt, slt, seq, crc, md5, sha1, itemStatus, trim, single, root, maxDegreeOfParallelism, _logger);
} }
/// <summary>
/// Wrap verifying files using an input DAT
/// </summary>
/// <param name="datfiles">Names of the DATs to compare against</param>
/// <param name="inputs">Input directories to compare against</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
private static void InitVerify(List<string> datfiles, List<string> inputs, string tempDir, string headerToCheckAgainst)
{
// Get the archive scanning level
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
DateTime start = DateTime.Now;
_logger.User("Populating internal DAT...");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, _logger, keep: true, softlist: true);
}
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
FileTools.VerifyDirectory(datdata, inputs, tempDir, headerToCheckAgainst, _logger);
}
#endregion #endregion
} }
} }

View File

@@ -981,8 +981,8 @@ namespace SabreTools
// If we're using the sorter // If we're using the sorter
else if (sort) else if (sort)
{ {
InitSortVerify(datfiles, inputs, outDir, tempDir, quickScan, addFileDates, toFolder, InitSort(datfiles, inputs, outDir, tempDir, quickScan, addFileDates, toFolder,
verify, delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header); delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header);
} }
// Split a DAT by extension // Split a DAT by extension
@@ -1021,8 +1021,7 @@ namespace SabreTools
// If we're using the verifier // If we're using the verifier
else if (verify) else if (verify)
{ {
InitSortVerify(datfiles, inputs, outDir, tempDir, quickScan, addFileDates, toFolder, InitVerify(datfiles, inputs, tempDir, header);
verify, delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header);
} }
// If nothing is set, show the help // If nothing is set, show the help

View File

@@ -312,8 +312,8 @@ namespace SabreTools
{ {
if (datfiles.Count > 0) if (datfiles.Count > 0)
{ {
InitSortVerify(datfiles, inputs, outDir, tempDir, quickScan, date, toFolder, InitSort(datfiles, inputs, outDir, tempDir, quickScan, date, toFolder,
verify, delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header, logger); delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header, logger);
} }
else else
{ {
@@ -387,7 +387,7 @@ namespace SabreTools
logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false,
false, false, delete, tgz, romba, asl, false, null, logger); false, delete, tgz, romba, asl, false, null, logger);
return ss.Convert(); return ss.Convert();
} }
@@ -402,7 +402,6 @@ namespace SabreTools
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="sevenzip">Integer representing the archive handling level for 7z</param> /// <param name="sevenzip">Integer representing the archive handling level for 7z</param>
/// <param name="toFolder">True if files should be output to folder, false otherwise</param> /// <param name="toFolder">True if files should be output to folder, false otherwise</param>
/// <param name="verify">True if output directory should be checked instead of rebuilt to, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="tgz">True to output files in TorrentGZ format, false for TorrentZip</param> /// <param name="tgz">True to output files in TorrentGZ format, false for TorrentZip</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param> /// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
@@ -412,8 +411,8 @@ namespace SabreTools
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param> /// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
private static void InitSortVerify(List<string> datfiles, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date, private static void InitSort(List<string> datfiles, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
bool toFolder, bool verify, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst, Logger logger) bool toFolder, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst, Logger logger)
{ {
// Get the archive scanning level // Get the archive scanning level
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip);
@@ -430,8 +429,35 @@ namespace SabreTools
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date,
toFolder, verify, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, logger); toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, logger);
ss.StartProcessing(); ss.RebuildToOutput();
}
/// <summary>
/// Wrap verifying files using an input DAT
/// </summary>
/// <param name="datfiles">Names of the DATs to compare against</param>
/// <param name="inputs">Input directories to compare against</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
private static void InitVerify(List<string> datfiles, List<string> inputs, string tempDir, string headerToCheckAgainst, Logger logger)
{
// Get the archive scanning level
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
DateTime start = DateTime.Now;
logger.User("Populating internal DAT...");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, logger, keep: true, softlist: true);
}
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
FileTools.VerifyDirectory(datdata, inputs, tempDir, headerToCheckAgainst, logger);
} }
} }
} }