diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index f1400fba..574a608a 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -148,13 +148,19 @@ namespace SabreTools.Helper.Data helptext.Add(" -?, -h, --help Show this help"); // Convert - helptext.Add(" -cv, --convert Enable conversion of input files to TZip"); + helptext.Add(" -cv, --convert Enable conversion of input files to folder"); helptext.Add(" -dat= DAT to be used as a filter for conversion"); helptext.Add(" -out= Output directory"); helptext.Add(" -t=, --temp= Set the temporary directory to use"); helptext.Add(" -del, --delete Delete input files [DO NOT USE]"); + //helptext.Add(" -t7z Enable Torrent7z output"); + //helptext.Add(" -tar Enable TAR output"); helptext.Add(" -tgz Enable TorrentGZ output"); helptext.Add(" -r, --romba Enable Romba depot dir output"); + //helptext.Add(" -tlrz Enable TorrentLRZ output"); + //helptext.Add(" -trar Enable TorrentRAR output"); + //helptext.Add(" -txz Enable TorrentXZ output"); + helptext.Add(" -tzip Enable TorrentZip output"); helptext.Add(" -7z={0} Set scanning level for 7z archives"); helptext.Add(" -gz={2} Set scanning level for GZip archives"); helptext.Add(" -rar={2} Set scanning level for RAR archives"); @@ -221,9 +227,14 @@ namespace SabreTools.Helper.Data helptext.Add(" -del, --delete Delete input files [DO NOT USE]"); helptext.Add(" -qs, --quick Enable quick scanning of archives"); helptext.Add(" -ad, --add-date Add original dates from DAT, if possible"); + //helptext.Add(" -t7z Enable Torrent7z output"); + //helptext.Add(" -tar Enable TAR output"); helptext.Add(" -tgz Enable TorrentGZ output"); - helptext.Add(" -r, --romba Enable Romba depot dir output"); - helptext.Add(" -do, --directory Output files as uncompressed"); + helptext.Add(" -r, --romba Enable Romba depot dir output"); + //helptext.Add(" -tlrz Enable TorrentLRZ output"); + //helptext.Add(" -trar Enable TorrentRAR output"); + //helptext.Add(" -txz Enable TorrentXZ output"); + helptext.Add(" -tzip Enable TorrentZip output"); helptext.Add(" -h=, --header= Set a header skipper to use, blank means all"); helptext.Add(" -7z={0} Set scanning level for 7z archives"); helptext.Add(" -gz={2} Set scanning level for GZip archives"); diff --git a/SabreTools.Helper/Dats/DatFile.cs b/SabreTools.Helper/Dats/DatFile.cs index 5922e42d..ff4a0041 100644 --- a/SabreTools.Helper/Dats/DatFile.cs +++ b/SabreTools.Helper/Dats/DatFile.cs @@ -4279,17 +4279,27 @@ namespace SabreTools.Helper.Dats /// List of inputs to convert over to TorrentZip or TorrentGZ /// Output folder to rebuild to, blank is the current directory /// Temporary directory to use in file extraction - /// True if files should be output in TorrentGZ format, false for TorrentZip + /// Output format that files should be written to /// True if TorrentGZ files should be output in romba depot format, false otherwise /// True if input files should be deleted, false otherwise /// ArchiveScanLevel representing how files should be treated /// Logger object for file and console output /// True if processing was a success, false otherwise - public bool ConvertFiles(List inputs, string outDir, string tempDir, bool tgz, + public bool ConvertFiles(List inputs, string outDir, string tempDir, OutputFormat outputFormat, bool romba, bool delete, ArchiveScanLevel archiveScanLevel, Logger logger) { bool success = true; + // Check to see that there's an output directory and a temp directory defined + if (String.IsNullOrEmpty(outDir)) + { + outDir = "output"; + } + if (String.IsNullOrEmpty(tempDir)) + { + tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + } + // First, check that the output directory exists if (!Directory.Exists(outDir)) { @@ -4332,13 +4342,17 @@ namespace SabreTools.Helper.Dats logger.User("Processing file " + input); - if (tgz) + switch (outputFormat) { - success &= ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); - } - else - { - success &= ArchiveTools.WriteToArchive(input, outDir, rom, logger); + case OutputFormat.TorrentGzip: + success &= ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); + break; + case OutputFormat.TorrentZip: + success &= ArchiveTools.WriteTorrentZip(input, outDir, rom, logger); + break; + case OutputFormat.Folder: + default: + break; } } @@ -4365,15 +4379,19 @@ namespace SabreTools.Helper.Dats } } - logger.User("Processing file " + input); + logger.User("Processing file " + file); - if (tgz) + switch (outputFormat) { - success &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger); - } - else - { - success &= ArchiveTools.WriteToArchive(file, outDir, rom, logger); + case OutputFormat.TorrentGzip: + success &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger); + break; + case OutputFormat.TorrentZip: + success &= ArchiveTools.WriteTorrentZip(file, outDir, rom, logger); + break; + case OutputFormat.Folder: + default: + break; } } @@ -4443,9 +4461,8 @@ namespace SabreTools.Helper.Dats /// Temporary directory for archive extraction /// True to enable external scanning of archives, false otherwise /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip + /// Output format that files should be written to /// True if files should be output in Romba depot folders, false otherwise /// ArchiveScanLevel representing the archive handling levels /// True if the updated DAT should be output, false otherwise @@ -4453,7 +4470,7 @@ namespace SabreTools.Helper.Dats /// Logger object for file and console output /// True if rebuilding was a success, false otherwise public bool RebuildToOutput(List inputs, string outDir, string tempDir, bool quickScan, bool date, - bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, + bool delete, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { #region Perform setup @@ -4760,7 +4777,33 @@ namespace SabreTools.Helper.Dats #region Rebuild games in order - logger.User("Rebuilding all files to " + (toFolder ? "directory" : (tgz ? "TorrentGZ" : "TorrentZip"))); + switch (outputFormat) + { + case OutputFormat.Folder: + logger.User("Rebuilding all files to directory"); + break; + case OutputFormat.TapeArchive: + logger.User("Rebuilding all files to TAR"); + break; + case OutputFormat.Torrent7Zip: + logger.User("Rebuilding all files to Torrent7Z"); + break; + case OutputFormat.TorrentGzip: + logger.User("Rebuilding all files to TorrentGZ"); + break; + case OutputFormat.TorrentLrzip: + logger.User("Rebuilding all files to TorrentLRZ"); + break; + case OutputFormat.TorrentRar: + logger.User("Rebuilding all files to TorrentRAR"); + break; + case OutputFormat.TorrentXZ: + logger.User("Rebuilding all files to TorrentXZ"); + break; + case OutputFormat.TorrentZip: + logger.User("Rebuilding all files to TorrentZip"); + break; + } start = DateTime.Now; // Now loop through the keys and create the correct output items @@ -4818,37 +4861,47 @@ namespace SabreTools.Helper.Dats } // And now rebuild accordingly - if (toFolder) + switch (outputFormat) { - for (int i = 0; i < romsInGame.Count; i++) - { - string infile = pathsToFiles[i]; - Rom outrom = romsInGame[i]; - string outfile = Path.Combine(outDir, outrom.Machine.Name, outrom.Machine.Name); - - // Make sure the output folder is created - Directory.CreateDirectory(Path.GetDirectoryName(outfile)); - - // Now copy the file over - try + case OutputFormat.Folder: + for (int i = 0; i < romsInGame.Count; i++) { - File.Copy(infile, outfile); + string infile = pathsToFiles[i]; + Rom outrom = romsInGame[i]; + string outfile = Path.Combine(outDir, outrom.Machine.Name, outrom.Machine.Name); + + // Make sure the output folder is created + Directory.CreateDirectory(Path.GetDirectoryName(outfile)); + + // Now copy the file over + try + { + File.Copy(infile, outfile); + } + catch { } } - catch { } - } - } - else if (tgz) - { - for (int i = 0; i < itemsInGame.Count; i++) - { - string infile = pathsToFiles[i]; - Rom outrom = romsInGame[i]; - ArchiveTools.WriteTorrentGZ(infile, outDir, romba, logger); - } - } - else - { - ArchiveTools.WriteToArchive(pathsToFiles, outDir, romsInGame, logger); + break; + case OutputFormat.TapeArchive: + break; + case OutputFormat.Torrent7Zip: + break; + case OutputFormat.TorrentGzip: + for (int i = 0; i < itemsInGame.Count; i++) + { + string infile = pathsToFiles[i]; + Rom outrom = romsInGame[i]; + ArchiveTools.WriteTorrentGZ(infile, outDir, romba, logger); + } + break; + case OutputFormat.TorrentLrzip: + break; + case OutputFormat.TorrentRar: + break; + case OutputFormat.TorrentXZ: + break; + case OutputFormat.TorrentZip: + ArchiveTools.WriteTorrentZip(pathsToFiles, outDir, romsInGame, logger); + break; } // And now clear the temp folder to get rid of any transient files diff --git a/SabreTools.Helper/README.1ST b/SabreTools.Helper/README.1ST index 2eea9206..a994f961 100644 --- a/SabreTools.Helper/README.1ST +++ b/SabreTools.Helper/README.1ST @@ -140,10 +140,8 @@ Options: -?, -h, --help Show the built-in help text Built-in to most of the programs is a basic help text - -cv, --convert Enable conversion of input files to TorrentZip - Using a folder or set of folders, convert the input archives to TorrentZip, rebuilding - to another folder. This is useful for rebuilding non-TorrentZipped sets without having - to run them through a larger tool or helping to rebuild a romba depot + -cv, --convert Enable conversion of input files to unarchived folders + Using a folder or set of folders, rebuild to another folder. -dat= Name of the DAT to be used as a filter A supplied DAT file to be used as a filter in conversion. If a file is found in the @@ -163,10 +161,21 @@ Options: This is a WIP flag that allows for deletion of input files once they have been rebuilt. It is not recommended for normal use because it does not discriminate whether or not the input files were rebuilt or not before deletion + + -t7z Enable Torrent 7zip output [NOT IMPLEMENTED] + Instead of ouputting the files to folder, files will be rebuilt to Torrent7Zip (T7Z) + files. This format is based on the LZMA container format 7zip, but with custom header + information. This is currently unused by any major application. + -tar Enable Tape ARchive output [NOT IMPLEMENTED] + Instead of outputting the fiels to folder, files will be rebuilt to Tape ARchive (TAR) + files. This format is a standardized storage archive without any compression, usually + used with other compression formats around it. It is widely used in backup applications + and source code archives. + -tgz Enable Torrent GZ output - Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ - (TGZ) files. This format is based on the GZip archive format, but with custom header + Instead of outputting the files to folder, files will be rebuilt to TorrentGZ (TGZ) + files. This format is based on the GZip archive format, but with custom header information and a file name replaced by the SHA-1 of the file inside. This is primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but may be used more widely in the future. @@ -178,6 +187,28 @@ Options: includes two auxilary files, .romba_size and .romba_size.backup, that have the compressed size of the folder inside for use with Romba. + -tlrz Enable Torrent Long-Range Zip output [NOT IMPLEMENTED] + Instead of ouputting the files to folder, files will be rebuilt to Torrent Long-Range + Zip (TLRZ) files. This format is based on the LRZip file format as defined at + https://github.com/ckolivas/lrzip but with custom header information. This is currently + unused by any major application. + + -trar Enable Torrent RAR output [NOT IMPLEMENTED] + Instead of outputting files to folder, files will be rebuilt to Torrent RAR (TRAR) + files. This format is based on the RAR propietary format but with custom header + information. This is currently unused by any major application; + + -txz Enable Torrent XZ output [NOT IMPLEMENTED] + Instead of outputting files to folder, files will be rebuilt to Torrent XZ (TXZ) files. + This format is based on the LZMA container format XZ, but with custom header + information. This is currently unused by any major application; + + -tzip Enable Torrent Zip output + Instead of ouputting files to folder, files will be rebuilt to TorrentZip (TZ) files. + This format is based on the ZIP archive format, but with custom header information. + This is primarily used by external tool RomVault (http://www.romvault.com/) and is + already widely used. + -7z={0} Set scanning level for 7z archives -gz={2} Set scanning level for GZip archives -rar={2} Set scanning level for RAR archives @@ -389,9 +420,7 @@ Options: -ss, --sort Sort input files by a set of DATs This feature allows the user to quickly rebuild based on a supplied DAT file(s). By - default all files will be rebuilt to TorrentZip (TZ) files. This format is based on - the ZIP archive format, but with custom header information. This is primarily used by - external tool RomVault (http://www.romvault.com/) and is already widely used. + default all files will be rebuilt to uncompressed folders in the output directory. -dat= Name of the DAT to be used for the various options The user-supplied DAT used to check which files need to be rebuilt. Multiple @@ -423,13 +452,24 @@ Options: invalidate the output files as proper TorrentZip files because the date will not match the standard. + -t7z Enable Torrent 7zip output [NOT IMPLEMENTED] + Instead of ouputting the files to folder, files will be rebuilt to Torrent7Zip (T7Z) + files. This format is based on the LZMA container format 7zip, but with custom header + information. This is currently unused by any major application. + + -tar Enable Tape ARchive output [NOT IMPLEMENTED] + Instead of outputting the fiels to folder, files will be rebuilt to Tape ARchive (TAR) + files. This format is a standardized storage archive without any compression, usually + used with other compression formats around it. It is widely used in backup applications + and source code archives. + -tgz Enable Torrent GZ output - Instead of outputting the files to ZIP archives, files will be rebuilt to TorrentGZ - (TGZ) files. This format is based on the GZip archive format, but with custom header + Instead of outputting the files to folder, files will be rebuilt to TorrentGZ (TGZ) + files. This format is based on the GZip archive format, but with custom header information and a file name replaced by the SHA-1 of the file inside. This is primarily used by external tool Romba (https://github.com/uwedeportivo/romba), but may be used more widely in the future. - + -r, --romba Enable Romba depot directory output As an extension of the parent flag, this outputs the TGZ files into directories based on the structure used by Romba. This uses nested folders using the first @@ -437,10 +477,27 @@ Options: includes two auxilary files, .romba_size and .romba_size.backup, that have the compressed size of the folder inside for use with Romba. - -do, --directory Enable outputting files uncompressed - Instead of outputting the files to ZIP archives, files will be rebuilt to named - subdirectories within the output folder. This is useful for when the DAT does not - already have the flag specified. + -tlrz Enable Torrent Long-Range Zip output [NOT IMPLEMENTED] + Instead of ouputting the files to folder, files will be rebuilt to Torrent Long-Range + Zip (TLRZ) files. This format is based on the LRZip file format as defined at + https://github.com/ckolivas/lrzip but with custom header information. This is currently + unused by any major application. + + -trar Enable Torrent RAR output [NOT IMPLEMENTED] + Instead of outputting files to folder, files will be rebuilt to Torrent RAR (TRAR) + files. This format is based on the RAR propietary format but with custom header + information. This is currently unused by any major application; + + -txz Enable Torrent XZ output [NOT IMPLEMENTED] + Instead of outputting files to folder, files will be rebuilt to Torrent XZ (TXZ) files. + This format is based on the LZMA container format XZ, but with custom header + information. This is currently unused by any major application; + + -tzip Enable Torrent Zip output + Instead of ouputting files to folder, files will be rebuilt to TorrentZip (TZ) files. + This format is based on the ZIP archive format, but with custom header information. + This is primarily used by external tool RomVault (http://www.romvault.com/) and is + already widely used. -h=, --header= Remove headers from hash calculations If this is set, then all files that have copier headers that are detected will @@ -465,9 +522,10 @@ Options: that only 1 thread is to be used, it defaults to the original, serial implementation of the DFD code. - -ud, --update-dat Output updated DAT (rebuild only) - Once the files that were able to rebuilt are taken care of, a DAT of the files - that could not be matched will be output to the program directory. + -ud, --update-dat Output updated DAT (rebuild only) + Once the files that were able to rebuilt are taken care of, a DAT of the files + that could not be matched will be output to the program directory. + that could not be matched will be output to the program directory. -st, --stats Get statistics on all input DATs This will output by default the combined statistics for all input DAT files. The stats diff --git a/SabreTools.Helper/SabreTools.Helper.csproj b/SabreTools.Helper/SabreTools.Helper.csproj index 59ce4ab3..dee789d7 100644 --- a/SabreTools.Helper/SabreTools.Helper.csproj +++ b/SabreTools.Helper/SabreTools.Helper.csproj @@ -53,6 +53,7 @@ OnBuildSuccess + ..\packages\Mono.Data.Sqlite.Portable.1.0.3.5\lib\net4\Mono.Data.Sqlite.dll True diff --git a/SabreTools.Helper/Tools/ArchiveTools.cs b/SabreTools.Helper/Tools/ArchiveTools.cs index 73ff16b1..adddbe04 100644 --- a/SabreTools.Helper/Tools/ArchiveTools.cs +++ b/SabreTools.Helper/Tools/ArchiveTools.cs @@ -35,7 +35,7 @@ namespace SabreTools.Helper.Tools { string temp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); string realName = ExtractItem(inputArchive, sourceEntryName, temp, logger); - bool success = WriteToArchive(realName, outDir, destEntry, logger); + bool success = WriteTorrentZip(realName, outDir, destEntry, logger); Directory.Delete(temp, true); return success; } @@ -738,7 +738,7 @@ namespace SabreTools.Helper.Tools #region Writing /// - /// Copy a file to an output torrentzip archive + /// Write an input file to a tape archive /// /// Input filename to be moved /// Output directory to build to @@ -746,7 +746,7 @@ namespace SabreTools.Helper.Tools /// Logger object for file and console output /// True if the date from the DAT should be used if available, false otherwise (default) /// True if the archive was written properly, false otherwise - public static bool WriteToArchive(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + public static bool WriteTAR(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists List inputFiles = new List(); @@ -754,11 +754,11 @@ namespace SabreTools.Helper.Tools List roms = new List(); roms.Add(rom); - return WriteToArchive(inputFiles, outDir, roms, logger, date: date); + return WriteTAR(inputFiles, outDir, roms, logger, date: date); } /// - /// Copy a set of files to an output torrentzip archive (assuming the same output archive name) + /// Write a set of input files to a tape archive (assuming the same output archive name) /// /// Input filenames to be moved /// Output directory to build to @@ -766,7 +766,259 @@ namespace SabreTools.Helper.Tools /// Logger object for file and console output /// True if the date from the DAT should be used if available, false otherwise (default) /// True if the archive was written properly, false otherwise - public static bool WriteToArchive(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + public static bool WriteTAR(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + { + return false; + } + + /// + /// Write an input file to a torrent7z archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrent7Zip(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + { + // Wrap the individual inputs into lists + List inputFiles = new List(); + inputFiles.Add(inputFile); + List roms = new List(); + roms.Add(rom); + + return WriteTorrent7Zip(inputFiles, outDir, roms, logger, date: date); + } + + /// + /// Write a set of input files to a torrent7z archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrent7Zip(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + { + return false; + } + + /// + /// Write an input file to a torrent GZ file + /// + /// File to write from + /// Directory to write archive to + /// True if files should be output in Romba depot folders, false otherwise + /// Logger object for file and console output + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public static bool WriteTorrentGZ(string input, string outDir, bool romba, Logger logger) + { + // Check that the input file exists + if (!File.Exists(input)) + { + logger.Warning("File " + input + " does not exist!"); + return false; + } + input = Path.GetFullPath(input); + + // Make sure the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + } + outDir = Path.GetFullPath(outDir); + + // Now get the Rom info for the file so we have hashes and size + Rom rom = FileTools.GetFileInfo(input, logger); + + // Get the output file name + string outfile = Path.Combine(outDir, rom.SHA1 + ".gz"); + + // If the output file exists, don't try to write again + if (!File.Exists(outfile)) + { + // Compress the input stream + FileStream inputStream = File.OpenRead(input); + FileStream outputStream = File.Open(outfile, FileMode.Create, FileAccess.Write); + + // Open the output file for writing + BinaryWriter sw = new BinaryWriter(outputStream); + + // Write standard header and TGZ info + byte[] data = Constants.TorrentGZHeader + .Concat(Style.StringToByteArray(rom.MD5)) // MD5 + .Concat(Style.StringToByteArray(rom.CRC)) // CRC + .ToArray(); + sw.Write(data); + sw.Write((ulong)rom.Size); // Long size (Unsigned, Mirrored) + + // Now create a deflatestream from the input file + DeflateStream ds = new DeflateStream(outputStream, CompressionMode.Compress, CompressionLevel.BestCompression, true); + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + ds.Write(ibuffer, 0, ilen); + ds.Flush(); + } + ds.Dispose(); + + // Now write the standard footer + sw.Write(Style.StringToByteArray(rom.CRC).Reverse().ToArray()); + sw.Write((uint)rom.Size); + + // Dispose of everything + sw.Dispose(); + outputStream.Dispose(); + inputStream.Dispose(); + } + + // If we're in romba mode, create the subfolder and move the file + if (romba) + { + MoveToRombaFolder(rom, outDir, outfile, logger); + } + + return true; + } + + /// + /// Write an input file to a torrentlrzip archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentLRZ(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + { + // Wrap the individual inputs into lists + List inputFiles = new List(); + inputFiles.Add(inputFile); + List roms = new List(); + roms.Add(rom); + + return WriteTorrentLRZ(inputFiles, outDir, roms, logger, date: date); + } + + /// + /// Write a set of input files to a torrentlrzip archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentLRZ(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + { + return false; + } + + /// + /// Write an input file to a torrentrar archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentRAR(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + { + // Wrap the individual inputs into lists + List inputFiles = new List(); + inputFiles.Add(inputFile); + List roms = new List(); + roms.Add(rom); + + return WriteTorrentRAR(inputFiles, outDir, roms, logger, date: date); + } + + /// + /// Write a set of input files to a torrentrar archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentRAR(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + { + return false; + } + + /// + /// Write an input file to a torrentxz archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentXZ(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + { + // Wrap the individual inputs into lists + List inputFiles = new List(); + inputFiles.Add(inputFile); + List roms = new List(); + roms.Add(rom); + + return WriteTorrentXZ(inputFiles, outDir, roms, logger, date: date); + } + + /// + /// Write a set of input files to a torrentxz archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentXZ(List inputFiles, string outDir, List roms, Logger logger, bool date = false) + { + return false; + } + + /// + /// Write an input file to a torrentzip archive + /// + /// Input filename to be moved + /// Output directory to build to + /// RomData representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentZip(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) + { + // Wrap the individual inputs into lists + List inputFiles = new List(); + inputFiles.Add(inputFile); + List roms = new List(); + roms.Add(rom); + + return WriteTorrentZip(inputFiles, outDir, roms, logger, date: date); + } + + /// + /// Write a set of input files to a torrentzip archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if the archive was written properly, false otherwise + public static bool WriteTorrentZip(List inputFiles, string outDir, List roms, Logger logger, bool date = false) { bool success = false; string tempFile = Path.GetTempFileName(); @@ -981,88 +1233,6 @@ namespace SabreTools.Helper.Tools return true; } - /// - /// Write an input file to a torrent GZ file - /// - /// File to write from - /// Directory to write archive to - /// True if files should be output in Romba depot folders, false otherwise - /// Logger object for file and console output - /// True if the write was a success, false otherwise - /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. - public static bool WriteTorrentGZ(string input, string outDir, bool romba, Logger logger) - { - // Check that the input file exists - if (!File.Exists(input)) - { - logger.Warning("File " + input + " does not exist!"); - return false; - } - input = Path.GetFullPath(input); - - // Make sure the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - } - outDir = Path.GetFullPath(outDir); - - // Now get the Rom info for the file so we have hashes and size - Rom rom = FileTools.GetFileInfo(input, logger); - - // Get the output file name - string outfile = Path.Combine(outDir, rom.SHA1 + ".gz"); - - // If the output file exists, don't try to write again - if (!File.Exists(outfile)) - { - // Compress the input stream - FileStream inputStream = File.OpenRead(input); - FileStream outputStream = File.Open(outfile, FileMode.Create, FileAccess.Write); - - // Open the output file for writing - BinaryWriter sw = new BinaryWriter(outputStream); - - // Write standard header and TGZ info - byte[] data = Constants.TorrentGZHeader - .Concat(Style.StringToByteArray(rom.MD5)) // MD5 - .Concat(Style.StringToByteArray(rom.CRC)) // CRC - .ToArray(); - sw.Write(data); - sw.Write((ulong)rom.Size); // Long size (Unsigned, Mirrored) - - // Now create a deflatestream from the input file - DeflateStream ds = new DeflateStream(outputStream, CompressionMode.Compress, CompressionLevel.BestCompression, true); - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - ds.Write(ibuffer, 0, ilen); - ds.Flush(); - } - ds.Dispose(); - - // Now write the standard footer - sw.Write(Style.StringToByteArray(rom.CRC).Reverse().ToArray()); - sw.Write((uint)rom.Size); - - // Dispose of everything - sw.Dispose(); - outputStream.Dispose(); - inputStream.Dispose(); - } - - // If we're in romba mode, create the subfolder and move the file - if (romba) - { - MoveToRombaFolder(rom, outDir, outfile, logger); - } - - return true; - } - /// /// Get the romba path for a file based on the rom's SHA-1 /// diff --git a/SabreTools.sln b/SabreTools.sln index 82cd20b1..0cb1376b 100644 --- a/SabreTools.sln +++ b/SabreTools.sln @@ -7,8 +7,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools", "SabreTools\Sa EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.Helper", "SabreTools.Helper\SabreTools.Helper.csproj", "{225A1AFD-0890-44E8-B779-7502665C23A5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleSort", "SimpleSort\SimpleSort.csproj", "{7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RombaSharp", "RombaSharp\RombaSharp.csproj", "{4728D479-8CFB-43E9-8C63-4774C6D73200}" EndProject Global @@ -36,14 +34,6 @@ Global {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|Any CPU.Build.0 = Release|Any CPU {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|x64.ActiveCfg = Release|x64 {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|x64.Build.0 = Release|x64 - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|x64.ActiveCfg = Debug|x64 - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|x64.Build.0 = Debug|x64 - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|Any CPU.Build.0 = Release|Any CPU - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|x64.ActiveCfg = Release|x64 - {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|x64.Build.0 = Release|x64 {4728D479-8CFB-43E9-8C63-4774C6D73200}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4728D479-8CFB-43E9-8C63-4774C6D73200}.Debug|Any CPU.Build.0 = Debug|Any CPU {4728D479-8CFB-43E9-8C63-4774C6D73200}.Debug|x64.ActiveCfg = Debug|x64 diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index 6df2bc6d..d9e9b916 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -21,14 +21,14 @@ namespace SabreTools /// Output directory (empty for default directory) /// Temporary directory for archive extraction /// True if input files should be deleted, false otherwise - /// True to output files in TorrentGZ format, false for TorrentZip + /// Output format that files should be written to /// True if files should be output in Romba depot folders, false otherwise /// Integer representing the archive handling level for 7z /// Integer representing the archive handling level for GZip /// Integer representing the archive handling level for RAR /// Integer representing the archive handling level for Zip public static bool InitConvertFolder(List datfiles, List inputs, string outDir, string tempDir, bool delete, - bool tgz, bool romba, int sevenzip, int gz, int rar, int zip) + OutputFormat outputFormat, bool romba, int sevenzip, int gz, int rar, int zip) { // Get the archive scanning level ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); @@ -64,7 +64,7 @@ namespace SabreTools } _logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - return datdata.ConvertFiles(inputs, outDir, tempDir, tgz, romba, delete, asl, _logger); + return datdata.ConvertFiles(inputs, outDir, tempDir, outputFormat, romba, delete, asl, _logger); } /// @@ -307,19 +307,18 @@ namespace SabreTools /// Temporary directory for archive extraction /// True to enable external scanning of archives, false otherwise /// True if the date from the DAT should be used if available, false otherwise - /// Integer representing the archive handling level for 7z - /// True if files should be output to folder, false otherwise /// True if input files should be deleted, false otherwise - /// True to output files in TorrentGZ format, false for TorrentZip + /// Output format that files should be written to /// True if files should be output in Romba depot folders, false otherwise + /// Integer representing the archive handling level for 7z /// Integer representing the archive handling level for GZip /// Integer representing the archive handling level for RAR /// Integer representing the archive handling level for Zip /// True if the updated DAT should be output, false otherwise /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Integer representing the maximum amount of parallelization to be used - private static void InitSort(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, bool date, - bool toFolder, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst, + private static void InitSort(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, bool date, bool delete, + OutputFormat outputFormat, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism) { // Get the archive scanning level @@ -336,7 +335,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - datdata.RebuildToOutput(inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, + datdata.RebuildToOutput(inputs, outDir, tempDir, quickScan, date, delete, outputFormat, romba, asl, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger); } diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index 04058628..f36594ba 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -389,9 +389,26 @@ namespace SabreTools case "--softlist": softlist = true; break; + case "-t7z": + case "--t7z": + outputFormat = OutputFormat.Torrent7Zip; + break; + case "-tar": + case "--tar": + outputFormat = OutputFormat.TapeArchive; + break; case "-tgz": case "--tgz": tgz = true; + outputFormat = OutputFormat.TorrentGzip; + break; + case "-tlrz": + case "--tlrz": + outputFormat = OutputFormat.TorrentLrzip; + break; + case "-trar": + case "--trar": + outputFormat = OutputFormat.TorrentRar; break; case "-trim": case "--trim": @@ -401,6 +418,14 @@ namespace SabreTools case "--tsv": statDatFormat = StatDatFormat.TSV; break; + case "-txz": + case "--txz": + outputFormat = OutputFormat.TorrentXZ; + break; + case "-tzip": + case "--tzip": + outputFormat = OutputFormat.TorrentZip; + break; case "-ud": case "--update": update = true; @@ -908,7 +933,7 @@ namespace SabreTools // Convert a folder to TGZ or TorrentZip if (convert) { - InitConvertFolder(datfiles, inputs, outDir, tempDir, delete, tgz, romba, sevenzip, gz, rar, zip); + InitConvertFolder(datfiles, inputs, outDir, tempDir, delete, outputFormat, romba, sevenzip, gz, rar, zip); } // Create a DAT from a directory or set of directories diff --git a/SimpleSort/SimpleSort.cs b/SimpleSort/SimpleSort.cs index 2d132e60..1523fb41 100644 --- a/SimpleSort/SimpleSort.cs +++ b/SimpleSort/SimpleSort.cs @@ -390,7 +390,7 @@ namespace SabreTools } logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - return datdata.ConvertFiles(inputs, outDir, tempDir, tgz, romba, delete, asl, logger); + return datdata.ConvertFiles(inputs, outDir, tempDir, (tgz ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), romba, delete, asl, logger); } ///