diff --git a/README.MD b/README.MD index 7d376ce0..18633bca 100644 --- a/README.MD +++ b/README.MD @@ -46,24 +46,22 @@ The main tool of the SabreTools suite. Performs the majority of the core feature
  • Optionally merge the output DAT so no partial duplicates are included
    • +
  • SimpleSort: Convert, rebuild, and verify using DAT files (requested by many) + +
  • SingleGame: Trim DAT entries and optionally merge into a single game (requested by Kludge)
  • UncompressedSize: Get various statistics on a DAT or folder of DATs including number of roms, disks, and even total size in a human readable format (requested by Obiwantje and others)
    • This tool has a comprehensive list of command line parameters that can be used to do the above and much more. -SimpleSort -

    -A simple rebuild-from-DAT tool that allows users to sort from the commandline. -

    -

    Power User Tools

    This section is for tools that have been requested for a highly specific purpose and are not generally going to be used by regular users. All of them have a possibility of being included in the main DATabase program eventually. diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index f4105a10..b09fe20e 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -188,7 +188,7 @@ namespace SabreTools // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); - need.RebuildToOutput(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, 4, _logger); + need.RebuildToOutput(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, OutputFormat.TorrentGzip, true, asl, false, null, 4, _logger); } /// @@ -226,7 +226,7 @@ namespace SabreTools // Now scan all of those depots and rebuild ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); - datFile.RebuildToOutput(onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, 4, _logger); + datFile.RebuildToOutput(onlineDepots, outputFolder, _tmpdir, true, false, false, (copy ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), copy, asl, false, null, 4, _logger); } } diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index 574a608a..878c022d 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -383,41 +383,6 @@ namespace SabreTools.Helper.Data helptext.Add(" e.g. 8kb => 8000 or 8kib => 8192"); break; - case "SimpleSort": - helptext.Add(Resources.Resources.SimpleSort_Name + " - " + Resources.Resources.SimpleSort_Desc); - helptext.Add(barrier); - helptext.Add(Resources.Resources.Usage + ": " + Resources.Resources.SimpleSort_Name + " [options] [filename|dirname] ..."); - helptext.Add(""); - helptext.Add("Options:"); - helptext.Add(" -?, -h, --help Show this help"); - - // Convert, Sort, Verify - helptext.Add(" -dat= Input DAT to rebuild against (REQUIRED)"); - helptext.Add(" -out= Output directory"); - helptext.Add(" -t=, --temp= Set the temporary directory to use"); - helptext.Add(" -d, --delete Delete input files"); - helptext.Add(" -qs, --quick Enable quick scanning of archives"); - helptext.Add(" -ad, --add-date Add original dates from DAT, if possible"); - helptext.Add(" -v, --verify Enable verification of output directory"); - helptext.Add(" -c, --convert Enable conversion of input files to TGZ"); - helptext.Add(" Note: If a DAT is used, only files NOT included will rebuild"); - helptext.Add(" -tgz Enable TorrentGZ output"); - helptext.Add(" -r, --romba Enable Romba depot dir output"); - helptext.Add(" -do, --directory Output files as uncompressed"); - helptext.Add(" -h=, --header= Set a header skipper to use, blank means all"); - helptext.Add(" -7z={0} Set scanning level for 7z archives"); - helptext.Add(" -gz={2} Set scanning level for GZip archives"); - helptext.Add(" -rar={2} Set scanning level for RAR archives"); - helptext.Add(" -zip={0} Set scanning level for ZIP archives"); - helptext.Add(" -ud, --update-dat Output updated DAT"); - - // Additional Notes - helptext.Add(""); - helptext.Add("Archive scanning levels:"); - helptext.Add(" 0 Hash archive and contents"); - helptext.Add(" 1 Only hash contents"); - helptext.Add(" 2 Only hash archive"); - break; default: helptext.Add(Resources.Resources.Default_Desc); break; diff --git a/SabreTools.Helper/README.1ST b/SabreTools.Helper/README.1ST index a994f961..1e052aad 100644 --- a/SabreTools.Helper/README.1ST +++ b/SabreTools.Helper/README.1ST @@ -9,7 +9,6 @@ Table of Contents 2.0 Included Programs 2.1 RombaSharp 2.2 SabreTools - 2.3 SimpleSort 3.0 Removed Commands 4.0 Examples 5.0 Contributors @@ -123,6 +122,8 @@ Included within this tool are a few former standalone executables: - Filter: Filter a DAT based on various user-defined criteria, optionally using wildcards - HashSplit: Split a DAT based on the best available hash (No-dump, SHA-1, MD5, CRC) - MergeDAT: Merge and optionally dedupe an arbitrary number of DAT files + - SimpleSort: Convert input files to new output format, do a simple rebuild using a DAT, + and verify a folder using a given DAT - SingleGame: Trim game and rom names to fit NTFS length standards, optionally merging all roms to a single game named "!" and forcing unpack - UncompressedSize: Get statistics from one or more input DATs, including number of @@ -889,106 +890,6 @@ Options: defined, and that skipper exists, then it will be used instead of trying to find one that matches. -** Section 2.3 - SimpleSort - -SimpleSort is a WIP program that is meant as a command-line tool to quickly rebuild and -verify files based on a supplied DAT file. The eventual aim for this program is to have -a full rom management tool without a GUI, though this may not happen for a while. By default -all files will be rebuilt to TorrentZip (TZ) files. This format is based on the ZIP archive -format, but with custom header information. This is primarily used by external tool RomVault -(http://www.romvault.com/) and is already widely used. - -Usage: - SimpleSort.exe [options] [filename|dirname] ... - -Options: - -?, -h, --help Show the built-in help text - Built-in to most of the programs is a basic help text - - -dat= Name of the DAT to be used for the various options - This DAT file is required for everything that SimpleSort currently does, so not - supplying one will result in an error. Depending on the additional flags that are - supplied, the DAT will be used differently, as described below. By default, the - DAT is used to check which files need to be rebuilt. - - -out= Set the name of the output directory - This sets an output folder to be used by various parts of the program. As with the - DAT file, this input is used differently based on the flags that are supplied. By - default, the output folder is used as the target to rebuild files to. - - -t=, --temp= Set the name of the temporary directory - Optionally, a temp folder can be supplied in the case the default temp directory - (inside the running folder) is not preferred. This is used for any operations that - require an archive to be extracted. - - -d, --delete Enable deletion of the input files - Optionally, the input files, once processed, can be deleted. This can be useful - when the original file structure is no longer needed or if there is limited space - on the source drive. - - -qs, --quick Enable quick scanning of archives - For all archives, if this flag is enabled, it will only use the header information - to get the archive entries' file information. The upside to this is that it is much - quicker than extracting all files to the temp folder. On the downside, it can only - get the CRC and size from most archive formats, leading to possible issues. - - -ad, --add-date Write dates for each file parsed, if available - If this flag is set, the the date in the DAT will be used for the output file - instead of the standard date and time for TorrentZip. This will technically - invalidate the output files as proper TorrentZip files because the date will not - match the standard. - - -v, --verify Enable verification of output directory - This overrides the default rebuilding and only requires the DAT and the output folder. - Here, the DAT is used to verify the output directory directly and then output a - simple FixDAT. This can be misleading, currently, because it only checks for exact - matches. - - -c, --convert Enable filtering by input DAT - This overrides the default rebuilding by using the DAT as a filter of what not to - output. If no DAT is supplied, the entire input folder will be rebuild file-by-file - to the output folder. - - -tgz Enable Torrent GZ output - Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ - (TGZ) files. This format is based on the GZip archive format, but with custom header - information and a file name replaced by the SHA-1 of the file inside. This is - primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but - may be used more widely in the future. - - -r, --romba Enable Romba depot directory output - As an extension of the parent flag, this outputs the TGZ files into directories - based on the structure used by Romba. This uses nested folders using the first - 4 bytes of the SHA-1, 1 byte for each layer of the directory name. It also - includes two auxilary files, .romba_size and .romba_size.backup, that have the - compressed size of the folder inside for use with Romba. - - -do, --directory Enable outputting files uncompressed - Instead of outputting the files to ZIP archives, files will be rebuilt to named - subdirectories within the output folder. This is useful for when the DAT does not - already have the flag specified. - - -h=, --header= Remove headers from hash calculations - If this is set, then all files that have copier headers that are detected will - have them removed from the hash calculation. This will allow for a headered collection - to be hashed without possibly variant information. If a particular header skipper is - defined, and that skipper exists, then it will be used instead of trying to find one - that matches. - - -7z={0} Set scanning level for 7z archives - -gz={2} Set scanning level for GZip archives - -rar={2} Set scanning level for RAR archives - -zip={0} Set scanning level for ZIP archives - For each of the major archive types recognized by the libraries used by this - program, scan the archive in one of the following ways: - 0 Hash both archive and its contents - 1 Only hash contents of the archive - 2 Only hash archive itself (treat like a regular file) - - -ud, --update-dat Output updated DAT (rebuild only) - Once the files that were able to rebuilt are taken care of, a DAT of the files - that could not be matched will be output to the program directory. - ** Section 3.0 - Removed Commands In this section, there is a list of commands that have been retired and will no longer work. @@ -1402,12 +1303,99 @@ Below are originally from SabreTools / DATabase - subdirectories instead of archives. Below are originally from SimpleSort (Standalone) - + + -?, -h, --help Show the built-in help text + Built-in to most of the programs is a basic help text + + -dat= Name of the DAT to be used for the various options + This DAT file is required for everything that SimpleSort currently does, so not + supplying one will result in an error. Depending on the additional flags that are + supplied, the DAT will be used differently, as described below. By default, the + DAT is used to check which files need to be rebuilt. + -out= Set the name of the output directory + This sets an output folder to be used by various parts of the program. As with the + DAT file, this input is used differently based on the flags that are supplied. By + default, the output folder is used as the target to rebuild files to. + + -t=, --temp= Set the name of the temporary directory + Optionally, a temp folder can be supplied in the case the default temp directory + (inside the running folder) is not preferred. This is used for any operations that + require an archive to be extracted. + + -d, --delete Enable deletion of the input files + Optionally, the input files, once processed, can be deleted. This can be useful + when the original file structure is no longer needed or if there is limited space + on the source drive. + + -qs, --quick Enable quick scanning of archives + For all archives, if this flag is enabled, it will only use the header information + to get the archive entries' file information. The upside to this is that it is much + quicker than extracting all files to the temp folder. On the downside, it can only + get the CRC and size from most archive formats, leading to possible issues. + + -ad, --add-date Write dates for each file parsed, if available + If this flag is set, the the date in the DAT will be used for the output file + instead of the standard date and time for TorrentZip. This will technically + invalidate the output files as proper TorrentZip files because the date will not + match the standard. + + -v, --verify Enable verification of output directory + This overrides the default rebuilding and only requires the DAT and the output folder. + Here, the DAT is used to verify the output directory directly and then output a + simple FixDAT. This can be misleading, currently, because it only checks for exact + matches. + + -c, --convert Enable filtering by input DAT + This overrides the default rebuilding by using the DAT as a filter of what not to + output. If no DAT is supplied, the entire input folder will be rebuild file-by-file + to the output folder. + + -tgz Enable Torrent GZ output + Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ + (TGZ) files. This format is based on the GZip archive format, but with custom header + information and a file name replaced by the SHA-1 of the file inside. This is + primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but + may be used more widely in the future. + + -r, --romba Enable Romba depot directory output + As an extension of the parent flag, this outputs the TGZ files into directories + based on the structure used by Romba. This uses nested folders using the first + 4 bytes of the SHA-1, 1 byte for each layer of the directory name. It also + includes two auxilary files, .romba_size and .romba_size.backup, that have the + compressed size of the folder inside for use with Romba. + -tzip Enable TorrentZip output Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentZip (TZip) files. This format is based on the ZIP archive format, but with custom header information. This is primarily used by external tool RomVault (http://www.romvault.com/) and is already widely used. + + -do, --directory Enable outputting files uncompressed + Instead of outputting the files to ZIP archives, files will be rebuilt to named + subdirectories within the output folder. This is useful for when the DAT does not + already have the flag specified. + + -h=, --header= Remove headers from hash calculations + If this is set, then all files that have copier headers that are detected will + have them removed from the hash calculation. This will allow for a headered collection + to be hashed without possibly variant information. If a particular header skipper is + defined, and that skipper exists, then it will be used instead of trying to find one + that matches. + + -7z={0} Set scanning level for 7z archives + -gz={2} Set scanning level for GZip archives + -rar={2} Set scanning level for RAR archives + -zip={0} Set scanning level for ZIP archives + For each of the major archive types recognized by the libraries used by this + program, scan the archive in one of the following ways: + 0 Hash both archive and its contents + 1 Only hash contents of the archive + 2 Only hash archive itself (treat like a regular file) + + -ud, --update-dat Output updated DAT (rebuild only) + Once the files that were able to rebuilt are taken care of, a DAT of the files + that could not be matched will be output to the program directory. Below are originally from SingleGame (Standalone) - diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index f36594ba..657f0d71 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -88,8 +88,6 @@ namespace SabreTools single = false, softlist = false, superdat = false, - tgz = false, // SimpleSort - toFolder = false, // SimpleSort trim = false, skip = false, updateDat = false, // SimpleSort @@ -249,10 +247,6 @@ namespace SabreTools case "--diff-nd": diffMode |= DiffMode.NoDupes; break; - case "-do": - case "--directory": - toFolder = true; - break; case "-f": case "--files": parseArchivesAsFiles = true; @@ -399,7 +393,6 @@ namespace SabreTools break; case "-tgz": case "--tgz": - tgz = true; outputFormat = OutputFormat.TorrentGzip; break; case "-tlrz": @@ -974,8 +967,8 @@ namespace SabreTools // If we're using the sorter else if (sort) { - InitSort(datfiles, inputs, outDir, tempDir, quickScan, addFileDates, toFolder, - delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, header, maxParallelism); + InitSort(datfiles, inputs, outDir, tempDir, quickScan, addFileDates, delete, outputFormat, + romba, sevenzip, gz, rar, zip, updateDat, header, maxParallelism); } // Split a DAT by extension