diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index e151e4c9..4316a5c9 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using System.Web; using SabreTools.Library.Data; +using SabreTools.Library.FileTypes; using SabreTools.Library.Items; using SabreTools.Library.Skippers; using SabreTools.Library.Tools; @@ -3648,15 +3649,13 @@ namespace SabreTools.Library.DatFiles // If we don't have archives as files, try to scan the file as an archive if (!archivesAsFiles) { - // If all deep hash skip flags are set, do a quickscan - if (omitFromScan == Hash.SecureHashes) + // Get the base archive first + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(newItem); + + // Now get all extracted items from the archive + if (archive != null) { - extracted = ArchiveTools.GetArchiveFileInfo(newItem, date: addDate); - } - // Otherwise, get the list with whatever hashes are wanted - else - { - extracted = ArchiveTools.GetExtendedArchiveFileInfo(newItem, omitFromScan: omitFromScan, date: addDate); + extracted = archive.GetArchiveFileInfo(omitFromScan: omitFromScan, date: addDate); } } @@ -3687,7 +3686,18 @@ namespace SabreTools.Library.DatFiles // Then, if we're looking for blanks, get all of the blank folders and add them if (addBlanks) { - List empties = ArchiveTools.GetEmptyFoldersInArchive(newItem); + List empties = new List(); + + // Get the base archive first + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(newItem); + + // Now get all blank folders from the archive + if (archive != null) + { + empties = archive.GetEmptyFolders(); + } + + // Add add all of the found empties to the DAT Parallel.ForEach(empties, Globals.ParallelOptions, empty => { Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan); @@ -4211,22 +4221,20 @@ namespace SabreTools.Library.DatFiles if (shouldInternalProcess) { // Create an empty list of Roms for archive entries - List entries = new List(); + List entries = null; usedInternally = true; // Get the TGZ status for later bool isTorrentGzip = (ArchiveTools.GetTorrentGZFileInfo(file) != null); - // If we're in quickscan, use the header information - if (quickScan) - { - entries = ArchiveTools.GetArchiveFileInfo(file, date: date); - } - // Otherwise get the deeper information - else + // Get the base archive first + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(file); + + // Now get all extracted items from the archive + if (archive != null) { // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually - entries = ArchiveTools.GetExtendedArchiveFileInfo(file, omitFromScan: (quickScan ? Hash.SecureHashes : Hash.DeepHashes), date: date); + entries = archive.GetArchiveFileInfo(omitFromScan: (quickScan ? Hash.SecureHashes : Hash.DeepHashes), date: date); } // If the entries list is null, we encountered an error and should scan exteranlly @@ -4347,7 +4355,11 @@ namespace SabreTools.Library.DatFiles if (isZip != null) { string realName = null; - (fileStream, realName) = ArchiveTools.ExtractStream(file, datItem.Name); + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(file); + if (archive != null) + { + (fileStream, realName) = archive.ExtractEntryStream(datItem.Name); + } } // Otherwise, just open the filestream else @@ -4370,31 +4382,11 @@ namespace SabreTools.Library.DatFiles // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { - switch (outputFormat) - { - case OutputFormat.Folder: - rebuilt &= ArchiveTools.WriteFile(fileStream, outDir, item, date: date, overwrite: true); - break; - case OutputFormat.TapeArchive: - rebuilt &= ArchiveTools.WriteTAR(fileStream, outDir, (Rom)item, date: date); - break; - case OutputFormat.Torrent7Zip: - rebuilt &= ArchiveTools.WriteTorrent7Zip(fileStream, outDir, (Rom)item, date: date); - break; - case OutputFormat.TorrentGzip: - rebuilt &= ArchiveTools.WriteTorrentGZ(fileStream, outDir, romba); - break; - case OutputFormat.TorrentLrzip: - break; - case OutputFormat.TorrentRar: - break; - case OutputFormat.TorrentXZ: - rebuilt &= ArchiveTools.WriteTorrentXZ(fileStream, outDir, (Rom)item, date: date); - break; - case OutputFormat.TorrentZip: - rebuilt &= ArchiveTools.WriteTorrentZip(fileStream, outDir, (Rom)item, date: date); - break; - } + // Get the output archive, if possible + BaseArchive outputArchive = ArchiveTools.CreateArchiveFromOutputFormat(outputFormat); + + // Now rebuild to the output file + outputArchive.Write(fileStream, outDir, (Rom)item, date: date, romba: romba); } // Close the input stream @@ -4443,7 +4435,11 @@ namespace SabreTools.Library.DatFiles if (isZip != null) { string realName = null; - (fileStream, realName) = ArchiveTools.ExtractStream(file, datItem.Name); + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(file); + if (archive != null) + { + (fileStream, realName) = archive.ExtractEntryStream(datItem.Name); + } } // Otherwise, just open the filestream else @@ -4471,63 +4467,48 @@ namespace SabreTools.Library.DatFiles Globals.Logger.User("No matches found for '{0}', rebuilding accordingly from inverse flag...", Style.GetFileName(datItem.Name)); + // Get the output archive, if possible + BaseArchive outputArchive = ArchiveTools.CreateArchiveFromOutputFormat(outputFormat); + // Now rebuild to the output file - switch (outputFormat) + if (outputArchive == null) { - case OutputFormat.Folder: - string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.MachineName), item.Name); + string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.MachineName), item.Name); - // Make sure the output folder is created - Directory.CreateDirectory(Path.GetDirectoryName(outfile)); + // Make sure the output folder is created + Directory.CreateDirectory(Path.GetDirectoryName(outfile)); - // Now copy the file over - try + // Now copy the file over + try + { + FileStream writeStream = FileTools.TryCreate(outfile); + + // Copy the input stream to the output + int bufferSize = 4096 * 128; + byte[] ibuffer = new byte[bufferSize]; + int ilen; + while ((ilen = fileStream.Read(ibuffer, 0, bufferSize)) > 0) { - FileStream writeStream = FileTools.TryCreate(outfile); - - // Copy the input stream to the output - int bufferSize = 4096 * 128; - byte[] ibuffer = new byte[bufferSize]; - int ilen; - while ((ilen = fileStream.Read(ibuffer, 0, bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - writeStream.Dispose(); - - if (date && !String.IsNullOrEmpty(item.Date)) - { - File.SetCreationTime(outfile, DateTime.Parse(item.Date)); - } - - rebuilt &= true; + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); } - catch + writeStream.Dispose(); + + if (date && !String.IsNullOrEmpty(item.Date)) { - rebuilt &= false; + File.SetCreationTime(outfile, DateTime.Parse(item.Date)); } - break; - case OutputFormat.TapeArchive: - rebuilt &= ArchiveTools.WriteTAR(fileStream, outDir, item, date: date); - break; - case OutputFormat.Torrent7Zip: - rebuilt &= ArchiveTools.WriteTorrent7Zip(fileStream, outDir, item, date: date); - break; - case OutputFormat.TorrentGzip: - rebuilt &= ArchiveTools.WriteTorrentGZ(fileStream, outDir, romba); - break; - case OutputFormat.TorrentLrzip: - break; - case OutputFormat.TorrentRar: - break; - case OutputFormat.TorrentXZ: - rebuilt &= ArchiveTools.WriteTorrentXZ(fileStream, outDir, item, date: date); - break; - case OutputFormat.TorrentZip: - rebuilt &= ArchiveTools.WriteTorrentZip(fileStream, outDir, item, date: date); - break; + rebuilt &= true; + } + catch + { + rebuilt &= false; + } + } + else + { + rebuilt &= outputArchive.Write(fileStream, outDir, item, date: date, romba: romba); } // Close the input stream @@ -4544,7 +4525,11 @@ namespace SabreTools.Library.DatFiles if (isZip != null) { string realName = null; - (fileStream, realName) = ArchiveTools.ExtractStream(file, datItem.Name); + BaseArchive archive = ArchiveTools.CreateArchiveFromExistingInput(file); + if (archive != null) + { + (fileStream, realName) = archive.ExtractEntryStream(datItem.Name); + } } // Otherwise, just open the filestream else @@ -4598,37 +4583,13 @@ namespace SabreTools.Library.DatFiles // If either copy succeeds, then we want to set rebuilt to true bool eitherSuccess = false; - switch (outputFormat) - { - case OutputFormat.Folder: - eitherSuccess |= ArchiveTools.WriteFile(transformStream, outDir, item, date: date, overwrite: true); - eitherSuccess |= ArchiveTools.WriteFile(fileStream, outDir, datItem, date: date, overwrite: true); - break; - case OutputFormat.TapeArchive: - eitherSuccess |= ArchiveTools.WriteTAR(transformStream, outDir, (Rom)item, date: date); - eitherSuccess |= ArchiveTools.WriteTAR(fileStream, outDir, (Rom)datItem, date: date); - break; - case OutputFormat.Torrent7Zip: - eitherSuccess |= ArchiveTools.WriteTorrent7Zip(transformStream, outDir, (Rom)item, date: date); - eitherSuccess |= ArchiveTools.WriteTorrent7Zip(fileStream, outDir, (Rom)datItem, date: date); - break; - case OutputFormat.TorrentGzip: - eitherSuccess |= ArchiveTools.WriteTorrentGZ(transformStream, outDir, romba); - eitherSuccess |= ArchiveTools.WriteTorrentGZ(fileStream, outDir, romba); - break; - case OutputFormat.TorrentLrzip: - break; - case OutputFormat.TorrentRar: - break; - case OutputFormat.TorrentXZ: - eitherSuccess |= ArchiveTools.WriteTorrentXZ(transformStream, outDir, (Rom)item, date: date); - eitherSuccess |= ArchiveTools.WriteTorrentXZ(fileStream, outDir, (Rom)datItem, date: date); - break; - case OutputFormat.TorrentZip: - eitherSuccess |= ArchiveTools.WriteTorrentZip(transformStream, outDir, (Rom)item, date: date); - eitherSuccess |= ArchiveTools.WriteTorrentZip(fileStream, outDir, (Rom)datItem, date: date); - break; - } + + // Get the output archive, if possible + BaseArchive outputArchive = ArchiveTools.CreateArchiveFromOutputFormat(outputFormat); + + // Now rebuild to the output file + eitherSuccess |= outputArchive.Write(transformStream, outDir, (Rom)item, date: date, romba: romba); + eitherSuccess |= outputArchive.Write(fileStream, outDir, (Rom)datItem, date: date, romba: romba); // Now add the success of either rebuild rebuilt &= eitherSuccess; diff --git a/SabreTools.Library/FileTypes/BaseArchive.cs b/SabreTools.Library/FileTypes/BaseArchive.cs new file mode 100644 index 00000000..39858144 --- /dev/null +++ b/SabreTools.Library/FileTypes/BaseArchive.cs @@ -0,0 +1,132 @@ +using System.Collections.Generic; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; + +#if MONO +using System.IO; +#else +using MemoryStream = System.IO.MemoryStream; +using Stream = System.IO.Stream; +#endif +using SharpCompress.Common; + +namespace SabreTools.Library.FileTypes +{ + public abstract class BaseArchive + { + #region Protected instance variables + + // Buffer size used by archives + protected const int _bufferSize = 4096 * 128; + + protected ArchiveType _archiveType; + protected string _filename; + + #endregion + + #region Construtors + + /// + /// Create a new Archive with no base file + /// + public BaseArchive() + { + } + + /// + /// Create a new Archive from the given file + /// + /// Name of the file to use as an archive + public BaseArchive(string filename) + { + _filename = filename; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public abstract bool ExtractAll(string outDir); + + /// + /// Attempt to extract an entry from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public abstract string ExtractEntry(string entryName, string outDir); + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public abstract (MemoryStream, string) ExtractEntryStream(string entryName); + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public abstract List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false); + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public abstract List GetEmptyFolders(); + + #endregion + + #region Writing + + /// + /// Write an input file to an archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public abstract bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false); + + /// + /// Write an input stream to an archive + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public abstract bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false); + + /// + /// Write a set of input files to an archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public abstract bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false); + + #endregion + } +} diff --git a/SabreTools.Library/External/CHDFile.cs b/SabreTools.Library/FileTypes/CHDFile.cs similarity index 99% rename from SabreTools.Library/External/CHDFile.cs rename to SabreTools.Library/FileTypes/CHDFile.cs index 792da2c7..b5c3eaf8 100644 --- a/SabreTools.Library/External/CHDFile.cs +++ b/SabreTools.Library/FileTypes/CHDFile.cs @@ -11,7 +11,7 @@ using SeekOrigin = System.IO.SeekOrigin; using Stream = System.IO.Stream; #endif -namespace SabreTools.Library.External +namespace SabreTools.Library.FileTypes { /// /// This is code adapted from chd.h and chd.cpp in MAME diff --git a/SabreTools.Library/FileTypes/Folder.cs b/SabreTools.Library/FileTypes/Folder.cs new file mode 100644 index 00000000..0f8ceb53 --- /dev/null +++ b/SabreTools.Library/FileTypes/Folder.cs @@ -0,0 +1,303 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using PathFormat = Alphaleonis.Win32.Filesystem.PathFormat; +#else +using Alphaleonis.Win32.Filesystem; + +using BinaryReader = System.IO.BinaryReader; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SearchOption = System.IO.SearchOption; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using SharpCompress.Common; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a folder for reading and writing + /// + public class Folder : BaseArchive + { + #region Constructors + + /// + /// Create a new folder with no base file + /// + public Folder() + : base() + { + } + + /// + /// Create a new folder from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public Folder(string filename) + : base(filename) + { + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + // Copy all files from the current folder to the output directory recursively + try + { + // Make sure the folders exist + Directory.CreateDirectory(_filename); + Directory.CreateDirectory(outDir); + + Directory.Copy(_filename, outDir, true, PathFormat.FullPath); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return false; + } + + return true; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + string realentry = null; + + // Copy single file from the current folder to the output directory, if exists + try + { + // Make sure the folders exist + Directory.CreateDirectory(_filename); + Directory.CreateDirectory(outDir); + + // Get all files from the input directory + List files = FileTools.RetrieveFiles(_filename, new List()); + + // Now sort through to find the first file that matches + string match = files.Where(s => s.EndsWith(entryName)).FirstOrDefault(); + + // If we had a file, copy that over to the new name + if (!String.IsNullOrEmpty(match)) + { + realentry = match; + File.Copy(match, Path.Combine(outDir, entryName)); + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return realentry; + } + + return realentry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realentry = null; + + // Copy single file from the current folder to the output directory, if exists + try + { + // Make sure the folders exist + Directory.CreateDirectory(_filename); + + // Get all files from the input directory + List files = FileTools.RetrieveFiles(_filename, new List()); + + // Now sort through to find the first file that matches + string match = files.Where(s => s.EndsWith(entryName)).FirstOrDefault(); + + // If we had a file, copy that over to the new name + if (!String.IsNullOrEmpty(match)) + { + FileTools.TryOpenRead(match).CopyTo(ms); + realentry = match; + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + return (ms, realentry); + } + + return (ms, realentry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + throw new NotImplementedException(); + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + throw new NotImplementedException(); + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrent LRZip file + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + /// + /// Write an input stream to a torrent LRZip file + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + bool success = false; + + // If either input is null or empty, return + if (inputStream == null || rom == null || rom.Name == null) + { + return success; + } + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Set internal variables + FileStream outputStream = null; + + // Get the output folder name from the first rebuild rom + string fileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName), Style.RemovePathUnsafeCharacters(rom.Name)); + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(fileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(fileName)); + } + + // Overwrite output files by default + outputStream = FileTools.TryCreate(fileName); + + // If the output stream isn't null + if (outputStream != null) + { + // Copy the input stream to the output + inputStream.Seek(0, SeekOrigin.Begin); + int bufferSize = 4096 * 128; + byte[] ibuffer = new byte[bufferSize]; + int ilen; + while ((ilen = inputStream.Read(ibuffer, 0, bufferSize)) > 0) + { + outputStream.Write(ibuffer, 0, ilen); + outputStream.Flush(); + } + outputStream.Dispose(); + + if (rom.Type == ItemType.Rom) + { + if (date && !String.IsNullOrEmpty(((Rom)rom).Date)) + { + File.SetCreationTime(fileName, DateTime.Parse(((Rom)rom).Date)); + } + } + + success = true; + } + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + inputStream.Dispose(); + outputStream?.Dispose(); + } + + return success; + } + + /// + /// Write a set of input files to a torrent LRZip archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentGZipArchive.cs b/SabreTools.Library/FileTypes/TorrentGZipArchive.cs new file mode 100644 index 00000000..0fd51e3d --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentGZipArchive.cs @@ -0,0 +1,395 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using BinaryReader = System.IO.BinaryReader; +using BinaryWriter = System.IO.BinaryWriter; +using EndOfStreamException = System.IO.EndOfStreamException; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using Ionic.Zlib; +using SharpCompress.Common; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a TorrentGZip archive for reading and writing + /// + public class TorrentGZipArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentGZipArchive with no base file + /// + public TorrentGZipArchive() + : base() + { + } + + /// + /// Create a new TorrentGZipArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentGZipArchive(string filename) + : base(filename) + { + _archiveType = ArchiveType.GZip; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + bool encounteredErrors = true; + + try + { + // Create the temp directory + Directory.CreateDirectory(outDir); + + // Decompress the _filename stream + FileStream outstream = FileTools.TryCreate(Path.Combine(outDir, Path.GetFileNameWithoutExtension(_filename))); + GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(_filename), Ionic.Zlib.CompressionMode.Decompress); + gzstream.CopyTo(outstream); + + // Dispose of the streams + outstream.Dispose(); + gzstream.Dispose(); + + encounteredErrors = false; + } + catch (EndOfStreamException) + { + // Catch this but don't count it as an error because SharpCompress is unsafe + } + catch (InvalidOperationException) + { + encounteredErrors = true; + } + catch (Exception) + { + // Don't log file open errors + encounteredErrors = true; + } + + return encounteredErrors; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + // Try to extract a stream using the given information + (MemoryStream ms, string realEntry) = ExtractEntryStream(entryName); + + // If the memory stream and the entry name are both non-null, we write to file + if (ms != null && realEntry != null) + { + realEntry = Path.Combine(outDir, realEntry); + + // Create the output subfolder now + Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); + + // Now open and write the file if possible + FileStream fs = FileTools.TryCreate(realEntry); + if (fs != null) + { + ms.Seek(0, SeekOrigin.Begin); + byte[] zbuffer = new byte[_bufferSize]; + int zlen; + while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) + { + fs.Write(zbuffer, 0, zlen); + fs.Flush(); + } + + ms?.Dispose(); + fs?.Dispose(); + } + else + { + ms?.Dispose(); + fs?.Dispose(); + realEntry = null; + } + } + + return realEntry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realEntry = null; + + try + { + // Decompress the _filename stream + realEntry = Path.GetFileNameWithoutExtension(_filename); + GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(_filename), Ionic.Zlib.CompressionMode.Decompress); + + // Write the file out + byte[] gbuffer = new byte[_bufferSize]; + int glen; + while ((glen = gzstream.Read(gbuffer, 0, _bufferSize)) > 0) + { + + ms.Write(gbuffer, 0, glen); + ms.Flush(); + } + + // Dispose of the streams + gzstream.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + ms = null; + realEntry = null; + } + + return (ms, realEntry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + List found = new List(); + string gamename = Path.GetFileNameWithoutExtension(_filename); + + Rom possibleTgz = ArchiveTools.GetTorrentGZFileInfo(_filename); + + // If it was, then add it to the outputs and continue + if (possibleTgz != null && possibleTgz.Name != null) + { + found.Add(possibleTgz); + return found; + } + + try + { + // If secure hashes are disabled, do a quickscan + if (omitFromScan == Hash.SecureHashes) + { + Rom tempRom = new Rom(gamename, gamename, omitFromScan); + BinaryReader br = new BinaryReader(FileTools.TryOpenRead(_filename)); + br.BaseStream.Seek(-8, SeekOrigin.End); + byte[] headercrc = br.ReadBytes(4); + tempRom.CRC = BitConverter.ToString(headercrc.Reverse().ToArray()).Replace("-", string.Empty).ToLowerInvariant(); + byte[] headersize = br.ReadBytes(4); + tempRom.Size = BitConverter.ToInt32(headersize.Reverse().ToArray(), 0); + br.Dispose(); + + found.Add(tempRom); + } + // Otherwise, use the stream directly + else + { + GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(_filename), Ionic.Zlib.CompressionMode.Decompress); + Rom gzipEntryRom = (Rom)FileTools.GetStreamInfo(gzstream, gzstream.Length, omitFromScan: omitFromScan); + gzipEntryRom.Name = gzstream.FileName; + gzipEntryRom.MachineName = gamename; + gzipEntryRom.Date = (date && gzstream.LastModified != null ? gzstream.LastModified?.ToString("yyyy/MM/dd hh:mm:ss") : null); + found.Add(gzipEntryRom); + gzstream.Dispose(); + } + } + catch (Exception) + { + // Don't log file open errors + return null; + } + + return found; + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + // GZip files don't contain directories + return new List(); + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrent GZ file + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(string inputFile, string outDir, Rom rom = null, bool date = false, bool romba = false) + { + // Check that the input file exists + if (!File.Exists(inputFile)) + { + Globals.Logger.Warning("File '{0}' does not exist!", inputFile); + return false; + } + inputFile = Path.GetFullPath(inputFile); + + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date, romba); + } + + /// + /// Write an input stream to a torrent GZ file + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(Stream inputStream, string outDir, Rom rom = null, bool date = false, bool romba = false) + { + bool success = false; + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Make sure the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + } + outDir = Path.GetFullPath(outDir); + + // Now get the Rom info for the file so we have hashes and size + rom = (Rom)FileTools.GetStreamInfo(inputStream, inputStream.Length, keepReadOpen: true); + + // Get the output file name + string outfile = null; + + // If we have a romba output, add the romba path + if (romba) + { + outfile = Path.Combine(outDir, Style.GetRombaPath(rom.SHA1)); // TODO: When updating to SHA-256, this needs to update to SHA256 + + // Check to see if the folder needs to be created + if (!Directory.Exists(Path.GetDirectoryName(outfile))) + { + Directory.CreateDirectory(Path.GetDirectoryName(outfile)); + } + } + // Otherwise, we're just rebuilding to the main directory + else + { + outfile = Path.Combine(outDir, rom.SHA1 + ".gz"); // TODO: When updating to SHA-256, this needs to update to SHA256 + } + + // If the output file exists, don't try to write again + if (!File.Exists(outfile)) + { + // Compress the input stream + FileStream outputStream = FileTools.TryCreate(outfile); + + // Open the output file for writing + BinaryWriter sw = new BinaryWriter(outputStream); + + // Write standard header and TGZ info + byte[] data = Constants.TorrentGZHeader + .Concat(Style.StringToByteArray(rom.MD5)) // MD5 + .Concat(Style.StringToByteArray(rom.CRC)) // CRC + .ToArray(); + sw.Write(data); + sw.Write((ulong)rom.Size); // Long size (Unsigned, Mirrored) + + // Now create a deflatestream from the input file + DeflateStream ds = new DeflateStream(outputStream, Ionic.Zlib.CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true); + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + ds.Write(ibuffer, 0, ilen); + ds.Flush(); + } + ds.Dispose(); + + // Now write the standard footer + sw.Write(Style.StringToByteArray(rom.CRC).Reverse().ToArray()); + sw.Write((uint)rom.Size); + + // Dispose of everything + sw.Dispose(); + outputStream.Dispose(); + inputStream.Dispose(); + } + + return true; + } + + /// + /// Write a set of input files to a torrent GZ archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentLRZArchive.cs b/SabreTools.Library/FileTypes/TorrentLRZArchive.cs new file mode 100644 index 00000000..a0e62c00 --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentLRZArchive.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; + +#if MONO +using System.IO; +#else +using MemoryStream = System.IO.MemoryStream; +using Stream = System.IO.Stream; +#endif + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a TorrentLRZip archive for reading and writing + /// + /// TODO: LRZIP: https://github.com/ckolivas/lrzip + public class TorrentLRZArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentGZipArchive with no base file + /// + public TorrentLRZArchive() + : base() + { + } + + /// + /// Create a new TorrentGZipArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentLRZArchive(string filename) + : base(filename) + { + //_archiveType = ArchiveType.LRZip; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + throw new NotImplementedException(); + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + throw new NotImplementedException(); + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + throw new NotImplementedException(); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + throw new NotImplementedException(); + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + throw new NotImplementedException(); + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrent LRZip file + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + /// + /// Write an input stream to a torrent LRZip file + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + /// + /// Write a set of input files to a torrent LRZip archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentRARArchive.cs b/SabreTools.Library/FileTypes/TorrentRARArchive.cs new file mode 100644 index 00000000..f7f34be3 --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentRARArchive.cs @@ -0,0 +1,327 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using EndOfStreamException = System.IO.EndOfStreamException; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Common; +using SharpCompress.Readers; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a TorrentRAR archive for reading and writing + /// + public class TorrentRARArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentRARArchive with no base file + /// + public TorrentRARArchive() + : base() + { + } + + /// + /// Create a new TorrentRARArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentRARArchive(string filename) + : base(filename) + { + _archiveType = ArchiveType.Rar; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + bool encounteredErrors = true; + + try + { + // Create the temp directory + Directory.CreateDirectory(outDir); + + // Extract all files to the temp directory + RarArchive ra = RarArchive.Open(_filename); + foreach (RarArchiveEntry entry in ra.Entries) + { + entry.WriteToDirectory(outDir, new ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); + } + encounteredErrors = false; + ra.Dispose(); + } + catch (EndOfStreamException) + { + // Catch this but don't count it as an error because SharpCompress is unsafe + } + catch (InvalidOperationException) + { + encounteredErrors = true; + } + catch (Exception) + { + // Don't log file open errors + encounteredErrors = true; + } + + return encounteredErrors; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + // Try to extract a stream using the given information + (MemoryStream ms, string realEntry) = ExtractEntryStream(entryName); + + // If the memory stream and the entry name are both non-null, we write to file + if (ms != null && realEntry != null) + { + realEntry = Path.Combine(outDir, realEntry); + + // Create the output subfolder now + Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); + + // Now open and write the file if possible + FileStream fs = FileTools.TryCreate(realEntry); + if (fs != null) + { + ms.Seek(0, SeekOrigin.Begin); + byte[] zbuffer = new byte[_bufferSize]; + int zlen; + while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) + { + fs.Write(zbuffer, 0, zlen); + fs.Flush(); + } + + ms?.Dispose(); + fs?.Dispose(); + } + else + { + ms?.Dispose(); + fs?.Dispose(); + realEntry = null; + } + } + + return realEntry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realEntry = null; + + try + { + RarArchive ra = RarArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false, }); + foreach (RarArchiveEntry entry in ra.Entries) + { + if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) + { + // Write the file out + realEntry = entry.Key; + entry.WriteTo(ms); + } + } + ra.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + ms = null; + realEntry = null; + } + + return (ms, realEntry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + List found = new List(); + string gamename = Path.GetFileNameWithoutExtension(_filename); + + try + { + RarArchive ra = RarArchive.Open(FileTools.TryOpenRead(_filename)); + foreach (RarArchiveEntry entry in ra.Entries.Where(e => e != null && !e.IsDirectory)) + { + // If secure hashes are disabled, do a quickscan + if (omitFromScan == Hash.SecureHashes) + { + found.Add(new Rom + { + Type = ItemType.Rom, + Name = entry.Key, + Size = entry.Size, + CRC = entry.Crc.ToString("X").ToLowerInvariant(), + Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), + + MachineName = gamename, + }); + } + // Otherwise, use the stream directly + else + { + Stream entryStream = entry.OpenEntryStream(); + Rom rarEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); + rarEntryRom.Name = entry.Key; + rarEntryRom.MachineName = gamename; + rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); + found.Add(rarEntryRom); + entryStream.Dispose(); + } + } + + // Dispose of the archive + ra.Dispose(); + } + catch (Exception) + { + // Don't log file open errors + return null; + } + + return found; + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + List empties = new List(); + + try + { + RarArchive ra = RarArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false }); + List rarEntries = ra.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); + string lastRarEntry = null; + foreach (RarArchiveEntry entry in rarEntries) + { + if (entry != null) + { + // If the current is a superset of last, we skip it + if (lastRarEntry != null && lastRarEntry.StartsWith(entry.Key)) + { + // No-op + } + // If the entry is a directory, we add it + else if (entry.IsDirectory) + { + empties.Add(entry.Key); + lastRarEntry = entry.Key; + } + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + } + + return empties; + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrentrar archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date, romba); + } + + /// + /// Write an input stream to a torrentrar archive + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + /// + /// Write a set of input files to a torrentrar archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + throw new NotImplementedException(); + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentSevenZipArchive.cs b/SabreTools.Library/FileTypes/TorrentSevenZipArchive.cs new file mode 100644 index 00000000..b1f73fd5 --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentSevenZipArchive.cs @@ -0,0 +1,723 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using BinaryWriter = System.IO.BinaryWriter; +using EndOfStreamException = System.IO.EndOfStreamException; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using ROMVault2.SupportedFiles.Zip; +using SevenZip; +using SharpCompress.Archives; +using SharpCompress.Archives.SevenZip; +using SharpCompress.Common; +using SharpCompress.Readers; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a Torrent7zip archive for reading and writing + /// + /// TODO: Torrent 7-zip: https://sourceforge.net/p/t7z/code/HEAD/tree/ + public class TorrentSevenZipArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentSevenZipArchive with no base file + /// + public TorrentSevenZipArchive() + : base() + { + } + + /// + /// Create a new TorrentSevenZipArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentSevenZipArchive(string filename) + : base(filename) + { + _archiveType = ArchiveType.SevenZip; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + bool encounteredErrors = true; + + try + { + // Create the temp directory + Directory.CreateDirectory(outDir); + + // Extract all files to the temp directory + SevenZipArchive sza = SevenZipArchive.Open(FileTools.TryOpenRead(_filename)); + foreach (SevenZipArchiveEntry entry in sza.Entries) + { + entry.WriteToDirectory(outDir, new ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); + } + encounteredErrors = false; + sza.Dispose(); + } + catch (EndOfStreamException) + { + // Catch this but don't count it as an error because SharpCompress is unsafe + } + catch (InvalidOperationException) + { + encounteredErrors = true; + } + catch (Exception) + { + // Don't log file open errors + encounteredErrors = true; + } + + return encounteredErrors; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + // Try to extract a stream using the given information + (MemoryStream ms, string realEntry) = ExtractEntryStream(entryName); + + // If the memory stream and the entry name are both non-null, we write to file + if (ms != null && realEntry != null) + { + realEntry = Path.Combine(outDir, realEntry); + + // Create the output subfolder now + Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); + + // Now open and write the file if possible + FileStream fs = FileTools.TryCreate(realEntry); + if (fs != null) + { + ms.Seek(0, SeekOrigin.Begin); + byte[] zbuffer = new byte[_bufferSize]; + int zlen; + while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) + { + fs.Write(zbuffer, 0, zlen); + fs.Flush(); + } + + ms?.Dispose(); + fs?.Dispose(); + } + else + { + ms?.Dispose(); + fs?.Dispose(); + realEntry = null; + } + } + + return realEntry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realEntry = null; + + try + { + SevenZipArchive sza = SevenZipArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false, }); + foreach (SevenZipArchiveEntry entry in sza.Entries) + { + if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) + { + // Write the file out + realEntry = entry.Key; + entry.WriteTo(ms); + break; + } + } + sza.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + ms = null; + realEntry = null; + } + + return (ms, realEntry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + List found = new List(); + string gamename = Path.GetFileNameWithoutExtension(_filename); + + try + { + SevenZipArchive sza = SevenZipArchive.Open(FileTools.TryOpenRead(_filename)); + foreach (SevenZipArchiveEntry entry in sza.Entries.Where(e => e != null && !e.IsDirectory)) + { + // If secure hashes are disabled, do a quickscan + if (omitFromScan == Hash.SecureHashes) + { + found.Add(new Rom + { + Type = ItemType.Rom, + Name = entry.Key, + Size = entry.Size, + CRC = entry.Crc.ToString("X").ToLowerInvariant(), + Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), + + MachineName = gamename, + }); + } + // Otherwise, use the stream directly + else + { + Stream entryStream = entry.OpenEntryStream(); + Rom sevenZipEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); + sevenZipEntryRom.Name = entry.Key; + sevenZipEntryRom.MachineName = gamename; + sevenZipEntryRom.Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null); + found.Add(sevenZipEntryRom); + entryStream.Dispose(); + } + } + + // Dispose of the archive + sza.Dispose(); + } + catch (Exception) + { + // Don't log file open errors + return null; + } + + return found; + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + List empties = new List(); + + try + { + SevenZipArchive sza = SevenZipArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false }); + List sevenZipEntries = sza.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); + string lastSevenZipEntry = null; + foreach (SevenZipArchiveEntry entry in sevenZipEntries) + { + if (entry != null) + { + // If the current is a superset of last, we skip it + if (lastSevenZipEntry != null && lastSevenZipEntry.StartsWith(entry.Key)) + { + // No-op + } + // If the entry is a directory, we add it + else if (entry.IsDirectory) + { + empties.Add(entry.Key); + lastSevenZipEntry = entry.Key; + } + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + } + + return empties; + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrent7z archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); + } + + /// + /// Write an input file to a torrent7z archive + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either input is null or empty, return + if (inputStream == null || rom == null || rom.Name == null) + { + return success; + } + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Seek to the beginning of the stream + inputStream.Seek(0, SeekOrigin.Begin); + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".7z") ? "" : ".7z")); + + // Set internal variables + SevenZipBase.SetLibraryPath("7za.dll"); + SevenZipExtractor oldZipFile = null; + SevenZipCompressor zipFile; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(tempFile))) + { + Directory.CreateDirectory(Path.GetDirectoryName(tempFile)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = CompressionLevel.Normal, + }; + + // Create the temp directory + string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); + if (!Directory.Exists(tempPath)) + { + Directory.CreateDirectory(tempPath); + } + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(rom.Name, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + using (oldZipFile = new SevenZipExtractor(archiveFileName)) + { + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + + // If the old one doesn't contain the new file, then add it + if (!oldZipFile.ArchiveFileNames.Contains(rom.Name.Replace('\\', '/'))) + { + inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.FilesCount; i++) + { + inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = CompressionLevel.Normal, + }; + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(rom.Name, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, copy the file from the old archive + else + { + Stream oldZipFileEntryStream = new MemoryStream(); + oldZipFile.ExtractFile(index, oldZipFileEntryStream); + oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + oldZipFileEntryStream.Dispose(); + } + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + } + } + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + inputStream?.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + // Now make the file T7Z + // TODO: Add ACTUAL T7Z compatible code + + BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); + bw.Seek(0, SeekOrigin.Begin); + bw.Write(Constants.Torrent7ZipHeader); + bw.Seek(0, SeekOrigin.End); + + using (oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName))) + { + + // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) + byte[] tempsig = Constants.Torrent7ZipSignature; + if (oldZipFile.FilesCount > 1) + { + tempsig[16] = 0x2; + } + else + { + tempsig[16] = 0; + } + + bw.Write(tempsig); + bw.Flush(); + bw.Dispose(); + } + + return true; + } + + /// + /// Write a set of input files to a torrent7z archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either list of roms is null or empty, return + if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) + { + return success; + } + + // If the number of inputs is less than the number of available roms, return + if (inputFiles.Count < roms.Count) + { + return success; + } + + // If one of the files doesn't exist, return + foreach (string file in inputFiles) + { + if (!File.Exists(file)) + { + return success; + } + } + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".7z") ? "" : ".7z")); + + // Set internal variables + SevenZipBase.SetLibraryPath("7za.dll"); + SevenZipExtractor oldZipFile; + SevenZipCompressor zipFile; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = CompressionLevel.Normal, + }; + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); + } + + // Sort the keys in TZIP order + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Create the temp directory + string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); + if (!Directory.Exists(tempPath)) + { + Directory.CreateDirectory(tempPath); + } + + // Now add all of the files in order + foreach (string key in keys) + { + string newkey = Path.Combine(tempPath, key); + + File.Move(inputFiles[inputIndexMap[key]], newkey); + zipFile.CompressFiles(tempFile, newkey); + File.Move(newkey, inputFiles[inputIndexMap[key]]); + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + + FileTools.CleanDirectory(tempPath); + FileTools.TryDeleteDirectory(tempPath); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + using (oldZipFile = new SevenZipExtractor(archiveFileName)) + { + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + // If the old one contains the new file, then just skip out + if (oldZipFile.ArchiveFileNames.Contains(roms[i].Name.Replace('\\', '/'))) + { + continue; + } + + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.FilesCount; i++) + { + inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.SevenZip, + CompressionLevel = CompressionLevel.Normal, + }; + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + FileStream inputStream = FileTools.TryOpenRead(inputFiles[-index - 1]); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(key, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, copy the file from the old archive + else + { + Stream oldZipFileEntryStream = new MemoryStream(); + oldZipFile.ExtractFile(index, oldZipFileEntryStream); + oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + oldZipFileEntryStream.Dispose(); + } + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + } + } + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + // Now make the file T7Z + // TODO: Add ACTUAL T7Z compatible code + + BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); + bw.Seek(0, SeekOrigin.Begin); + bw.Write(Constants.Torrent7ZipHeader); + bw.Seek(0, SeekOrigin.End); + + using (oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName))) + { + // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) + byte[] tempsig = Constants.Torrent7ZipSignature; + if (oldZipFile.FilesCount > 1) + { + tempsig[16] = 0x2; + } + else + { + tempsig[16] = 0; + } + + bw.Write(tempsig); + bw.Flush(); + bw.Dispose(); + } + + return true; + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentTarArchive.cs b/SabreTools.Library/FileTypes/TorrentTarArchive.cs new file mode 100644 index 00000000..f8e3f56b --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentTarArchive.cs @@ -0,0 +1,636 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using EndOfStreamException = System.IO.EndOfStreamException; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using ROMVault2.SupportedFiles.Zip; +using SharpCompress.Archives; +using SharpCompress.Archives.Tar; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Writers; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a Torrent7zip archive for reading and writing + /// + /// TODO: Don't try to read entries to MemoryStream during write + public class TorrentTarArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentTarArchive with no base file + /// + public TorrentTarArchive() + : base() + { + } + + /// + /// Create a new TorrentTarArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentTarArchive(string filename) + : base(filename) + { + _archiveType = ArchiveType.Tar; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + bool encounteredErrors = true; + + try + { + // Create the temp directory + Directory.CreateDirectory(outDir); + + // Extract all files to the temp directory + TarArchive ta = TarArchive.Open(_filename); + foreach (TarArchiveEntry entry in ta.Entries) + { + entry.WriteToDirectory(outDir, new ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); + } + encounteredErrors = false; + ta.Dispose(); + } + catch (EndOfStreamException) + { + // Catch this but don't count it as an error because SharpCompress is unsafe + } + catch (InvalidOperationException) + { + encounteredErrors = true; + } + catch (Exception) + { + // Don't log file open errors + encounteredErrors = true; + } + + return encounteredErrors; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + // Try to extract a stream using the given information + (MemoryStream ms, string realEntry) = ExtractEntryStream(entryName); + + // If the memory stream and the entry name are both non-null, we write to file + if (ms != null && realEntry != null) + { + realEntry = Path.Combine(outDir, realEntry); + + // Create the output subfolder now + Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); + + // Now open and write the file if possible + FileStream fs = FileTools.TryCreate(realEntry); + if (fs != null) + { + ms.Seek(0, SeekOrigin.Begin); + byte[] zbuffer = new byte[_bufferSize]; + int zlen; + while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) + { + fs.Write(zbuffer, 0, zlen); + fs.Flush(); + } + + ms?.Dispose(); + fs?.Dispose(); + } + else + { + ms?.Dispose(); + fs?.Dispose(); + realEntry = null; + } + } + + return realEntry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realEntry = null; + + try + { + TarArchive ta = TarArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false, }); + foreach (TarArchiveEntry entry in ta.Entries) + { + if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) + { + // Write the file out + realEntry = entry.Key; + entry.WriteTo(ms); + } + } + ta.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + ms = null; + realEntry = null; + } + + return (ms, realEntry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + List found = new List(); + string gamename = Path.GetFileNameWithoutExtension(_filename); + + try + { + TarArchive ta = TarArchive.Open(FileTools.TryOpenRead(_filename)); + foreach (TarArchiveEntry entry in ta.Entries.Where(e => e != null && !e.IsDirectory)) + { + // If secure hashes are disabled, do a quickscan + if (omitFromScan == Hash.SecureHashes) + { + found.Add(new Rom + { + Type = ItemType.Rom, + Name = entry.Key, + Size = entry.Size, + CRC = entry.Crc.ToString("X").ToLowerInvariant(), + Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), + + MachineName = gamename, + }); + } + // Otherwise, use the stream directly + else + { + Stream entryStream = entry.OpenEntryStream(); + Rom tarEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); + tarEntryRom.Name = entry.Key; + tarEntryRom.MachineName = gamename; + tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); + found.Add(tarEntryRom); + entryStream.Dispose(); + } + } + + // Dispose of the archive + ta.Dispose(); + } + catch (Exception) + { + // Don't log file open errors + return null; + } + + return found; + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + List empties = new List(); + + try + { + TarArchive ta = TarArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false }); + List tarEntries = ta.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); + string lastTarEntry = null; + foreach (TarArchiveEntry entry in tarEntries) + { + if (entry != null) + { + // If the current is a superset of last, we skip it + if (lastTarEntry != null && lastTarEntry.StartsWith(entry.Key)) + { + // No-op + } + // If the entry is a directory, we add it + else if (entry.IsDirectory) + { + empties.Add(entry.Key); + lastTarEntry = entry.Key; + } + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + } + + return empties; + } + + #endregion + + #region Writing + + /// + /// Write an input file to a tape archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); + } + + /// + /// Write an input stream to a tape archive + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either input is null or empty, return + if (inputStream == null || rom == null || rom.Name == null) + { + return success; + } + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".tar") ? "" : ".tar")); + + // Set internal variables + TarArchive oldTarFile = TarArchive.Create(); + TarArchive tarFile = TarArchive.Create(); + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + // Get temporary date-time if possible + DateTime? usableDate = null; + if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out DateTime dt)) + { + usableDate = dt; + } + + // Copy the input stream to the output + inputStream.Seek(0, SeekOrigin.Begin); + tarFile.AddEntry(rom.Name, inputStream, size: rom.Size, modified: usableDate); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + oldTarFile = TarArchive.Open(archiveFileName); + + // Get a list of all current entries + List entries = oldTarFile.Entries.Select(i => i.Key).ToList(); + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + + // If the old one doesn't contain the new file, then add it + if (!entries.Contains(rom.Name.Replace('\\', '/'))) + { + inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); + } + + // Then add all of the old entries to it too + for (int i = 0; i < entries.Count; i++) + { + inputIndexMap.Add(entries[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= entries.Count) + { + success = true; + return success; + } + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // Get temporary date-time if possible + DateTime? usableDate = null; + if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out DateTime dt)) + { + usableDate = dt; + } + + // If we have the input file, add it now + if (index < 0) + { + // Copy the input file to the output + inputStream.Seek(0, SeekOrigin.Begin); + tarFile.AddEntry(rom.Name, inputStream, size: rom.Size, modified: usableDate); + } + + // Otherwise, copy the file from the old archive + else + { + // Get the stream from the original archive + TarArchiveEntry tae = oldTarFile.Entries.ElementAt(index); + MemoryStream entry = new MemoryStream(); + tae.OpenEntryStream().CopyTo(entry); + + // Copy the input stream to the output + tarFile.AddEntry(key, entry, size: tae.Size, modified: tae.LastModifiedTime); + } + } + } + + // Close the output tar file + tarFile.SaveTo(tempFile, new WriterOptions(CompressionType.None)); + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + inputStream.Dispose(); + tarFile.Dispose(); + oldTarFile.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + return success; + } + + /// + /// Write a set of input files to a tape archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either list of roms is null or empty, return + if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) + { + return success; + } + + // If the number of inputs is less than the number of available roms, return + if (inputFiles.Count < roms.Count) + { + return success; + } + + // If one of the files doesn't exist, return + foreach (string file in inputFiles) + { + if (!File.Exists(file)) + { + return success; + } + } + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".tar") ? "" : ".tar")); + + // Set internal variables + TarArchive oldTarFile = TarArchive.Create(); + TarArchive tarFile = TarArchive.Create(); + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); + } + + // Sort the keys in TZIP order + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Now add all of the files in order + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // Get temporary date-time if possible + DateTime? usableDate = null; + if (date && !String.IsNullOrEmpty(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out DateTime dt)) + { + usableDate = dt; + } + + // Copy the input stream to the output + tarFile.AddEntry(roms[index].Name, FileTools.TryOpenRead(inputFiles[index]), size: roms[index].Size, modified: usableDate); + } + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + oldTarFile = TarArchive.Open(archiveFileName); + + // Get a list of all current entries + List entries = oldTarFile.Entries.Select(i => i.Key).ToList(); + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + // If the old one contains the new file, then just skip out + if (entries.Contains(roms[i].Name.Replace('\\', '/'))) + { + continue; + } + + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); + } + + // Then add all of the old entries to it too + for (int i = 0; i < entries.Count; i++) + { + inputIndexMap.Add(entries[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= entries.Count) + { + success = true; + return success; + } + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + // Get temporary date-time if possible + DateTime? usableDate = null; + if (date && !String.IsNullOrEmpty(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out DateTime dt)) + { + usableDate = dt; + } + + // Copy the input file to the output + tarFile.AddEntry(roms[-index - 1].Name, FileTools.TryOpenRead(inputFiles[-index - 1]), size: roms[-index - 1].Size, modified: usableDate); + } + + // Otherwise, copy the file from the old archive + else + { + // Get the stream from the original archive + TarArchiveEntry tae = oldTarFile.Entries.ElementAt(index); + MemoryStream entry = new MemoryStream(); + tae.OpenEntryStream().CopyTo(entry); + + // Copy the input stream to the output + tarFile.AddEntry(key, entry, size: tae.Size, modified: tae.LastModifiedTime); + } + } + } + + // Close the output tar file + tarFile.SaveTo(tempFile, new WriterOptions(CompressionType.None)); + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + tarFile.Dispose(); + oldTarFile.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + return true; + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentXZArchive.cs b/SabreTools.Library/FileTypes/TorrentXZArchive.cs new file mode 100644 index 00000000..c415be7f --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentXZArchive.cs @@ -0,0 +1,555 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using BinaryWriter = System.IO.BinaryWriter; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using ROMVault2.SupportedFiles.Zip; +using SevenZip; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a TorrentXZ archive for reading and writing + /// + /// TODO: Wait for XZ write to be enabled by SevenZipSharp library + public class TorrentXZArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentGZipArchive with no base file + /// + public TorrentXZArchive() + : base() + { + } + + /// + /// Create a new TorrentGZipArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentXZArchive(string filename) + : base(filename) + { + //_archiveType = ArchiveType.XZip; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + throw new NotImplementedException(); + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + throw new NotImplementedException(); + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + throw new NotImplementedException(); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + throw new NotImplementedException(); + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + throw new NotImplementedException(); + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrent XZ file + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the write was a success, false otherwise + /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); + } + + /// + /// Write an input file to a torrent XZ archive + /// + /// Input stream to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either input is null or empty, return + if (inputStream == null || rom == null || rom.Name == null) + { + return success; + } + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Seek to the beginning of the stream + inputStream.Seek(0, SeekOrigin.Begin); + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".xz") ? "" : ".xz")); + + // Set internal variables + SevenZipBase.SetLibraryPath("7za.dll"); + SevenZipExtractor oldZipFile = null; + SevenZipCompressor zipFile; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(tempFile))) + { + Directory.CreateDirectory(Path.GetDirectoryName(tempFile)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.XZ, + CompressionLevel = CompressionLevel.Normal, + }; + + // Create the temp directory + string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); + if (!Directory.Exists(tempPath)) + { + Directory.CreateDirectory(tempPath); + } + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(rom.Name, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + using (oldZipFile = new SevenZipExtractor(archiveFileName)) + { + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + + // If the old one doesn't contain the new file, then add it + if (!oldZipFile.ArchiveFileNames.Contains(rom.Name.Replace('\\', '/'))) + { + inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.FilesCount; i++) + { + inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.XZ, + CompressionLevel = CompressionLevel.Normal, + }; + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(rom.Name, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, copy the file from the old archive + else + { + Stream oldZipFileEntryStream = new MemoryStream(); + oldZipFile.ExtractFile(index, oldZipFileEntryStream); + oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + oldZipFileEntryStream.Dispose(); + } + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + } + } + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + inputStream?.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + // Now make the file T7Z + // TODO: Add ACTUAL T7Z compatible code + + BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); + bw.Seek(0, SeekOrigin.Begin); + bw.Write(Constants.Torrent7ZipHeader); + bw.Seek(0, SeekOrigin.End); + + using (oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName))) + { + + // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) + byte[] tempsig = Constants.Torrent7ZipSignature; + if (oldZipFile.FilesCount > 1) + { + tempsig[16] = 0x2; + } + else + { + tempsig[16] = 0; + } + + bw.Write(tempsig); + bw.Flush(); + bw.Dispose(); + } + + return true; + } + + /// + /// Write a set of input files to a torrent XZ archive (assuming the same output archive name) + /// + /// Input files to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either list of roms is null or empty, return + if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) + { + return success; + } + + // If the number of inputs is less than the number of available roms, return + if (inputFiles.Count < roms.Count) + { + return success; + } + + // If one of the files doesn't exist, return + foreach (string file in inputFiles) + { + if (!File.Exists(file)) + { + return success; + } + } + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".xz") ? "" : ".xz")); + + // Set internal variables + SevenZipBase.SetLibraryPath("7za.dll"); + SevenZipExtractor oldZipFile; + SevenZipCompressor zipFile; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.XZ, + CompressionLevel = CompressionLevel.Normal, + }; + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); + } + + // Sort the keys in TZIP order + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Create the temp directory + string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); + if (!Directory.Exists(tempPath)) + { + Directory.CreateDirectory(tempPath); + } + + // Now add all of the files in order + foreach (string key in keys) + { + string newkey = Path.Combine(tempPath, key); + + File.Move(inputFiles[inputIndexMap[key]], newkey); + zipFile.CompressFiles(tempFile, newkey); + File.Move(newkey, inputFiles[inputIndexMap[key]]); + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + + FileTools.CleanDirectory(tempPath); + FileTools.TryDeleteDirectory(tempPath); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + using (oldZipFile = new SevenZipExtractor(archiveFileName)) + { + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + // If the old one contains the new file, then just skip out + if (oldZipFile.ArchiveFileNames.Contains(roms[i].Name.Replace('\\', '/'))) + { + continue; + } + + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.FilesCount; i++) + { + inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile = new SevenZipCompressor() + { + ArchiveFormat = OutArchiveFormat.XZ, + CompressionLevel = CompressionLevel.Normal, + }; + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + FileStream inputStream = FileTools.TryOpenRead(inputFiles[-index - 1]); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(key, inputStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + } + + // Otherwise, copy the file from the old archive + else + { + Stream oldZipFileEntryStream = new MemoryStream(); + oldZipFile.ExtractFile(index, oldZipFileEntryStream); + oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); + + // Create a stream dictionary + Dictionary dict = new Dictionary(); + dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); + + // Now add the stream + zipFile.CompressStreamDictionary(dict, tempFile); + oldZipFileEntryStream.Dispose(); + } + + // After the first file, make sure we're in append mode + zipFile.CompressionMode = CompressionMode.Append; + } + } + } + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + // Now make the file T7Z + // TODO: Add ACTUAL T7Z compatible code + + BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); + bw.Seek(0, SeekOrigin.Begin); + bw.Write(Constants.Torrent7ZipHeader); + bw.Seek(0, SeekOrigin.End); + + using (oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName))) + { + // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) + byte[] tempsig = Constants.Torrent7ZipSignature; + if (oldZipFile.FilesCount > 1) + { + tempsig[16] = 0x2; + } + else + { + tempsig[16] = 0; + } + + bw.Write(tempsig); + bw.Flush(); + bw.Dispose(); + } + + return true; + } + + #endregion + } +} diff --git a/SabreTools.Library/FileTypes/TorrentZipArchive.cs b/SabreTools.Library/FileTypes/TorrentZipArchive.cs new file mode 100644 index 00000000..ad40fb72 --- /dev/null +++ b/SabreTools.Library/FileTypes/TorrentZipArchive.cs @@ -0,0 +1,830 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using SabreTools.Library.Data; +using SabreTools.Library.Items; +using SabreTools.Library.Tools; + +#if MONO +using System.IO; +#else +using Alphaleonis.Win32.Filesystem; + +using EndOfStreamException = System.IO.EndOfStreamException; +using FileStream = System.IO.FileStream; +using MemoryStream = System.IO.MemoryStream; +using SeekOrigin = System.IO.SeekOrigin; +using Stream = System.IO.Stream; +#endif +using ROMVault2.SupportedFiles.Zip; +using SharpCompress.Common; +using SharpCompress.Readers; + +namespace SabreTools.Library.FileTypes +{ + /// + /// Represents a Torrent7zip archive for reading and writing + /// + public class TorrentZipArchive : BaseArchive + { + #region Constructors + + /// + /// Create a new TorrentZipArchive with no base file + /// + public TorrentZipArchive() + : base() + { + } + + /// + /// Create a new TorrentZipArchive from the given file + /// + /// Name of the file to use as an archive + /// True for opening file as read, false for opening file as write + public TorrentZipArchive(string filename) + : base(filename) + { + _archiveType = ArchiveType.Zip; + } + + #endregion + + #region Extraction + + /// + /// Attempt to extract a file as an archive + /// + /// Output directory for archive extraction + /// True if the extraction was a success, false otherwise + public override bool ExtractAll(string outDir) + { + bool encounteredErrors = true; + + try + { + // Create the temp directory + Directory.CreateDirectory(outDir); + + // Extract all files to the temp directory + ZipFile zf = new ZipFile(); + ZipReturn zr = zf.Open(_filename, new FileInfo(_filename).LastWriteTime.Ticks, true); + if (zr != ZipReturn.ZipGood) + { + throw new Exception(ZipFile.ZipErrorMessageText(zr)); + } + + for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) + { + // Open the read stream + zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); + + // Create the rest of the path, if needed + if (!String.IsNullOrEmpty(Path.GetDirectoryName(zf.Entries[i].FileName))) + { + Directory.CreateDirectory(Path.Combine(outDir, Path.GetDirectoryName(zf.Entries[i].FileName))); + } + + // If the entry ends with a directory separator, continue to the next item, if any + if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) + || zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) + || zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString())) + { + continue; + } + + FileStream writeStream = FileTools.TryCreate(Path.Combine(outDir, zf.Entries[i].FileName)); + + // If the stream is smaller than the buffer, just run one loop through to avoid issues + if (streamsize < _bufferSize) + { + byte[] ibuffer = new byte[streamsize]; + int ilen = readStream.Read(ibuffer, 0, (int)streamsize); + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + // Otherwise, we do the normal loop + else + { + int realBufferSize = (streamsize < _bufferSize ? (int)streamsize : _bufferSize); + byte[] ibuffer = new byte[realBufferSize]; + int ilen; + while ((ilen = readStream.Read(ibuffer, 0, realBufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + } + + zr = zf.CloseReadStream(); + writeStream.Dispose(); + } + zf.Close(); + encounteredErrors = false; + } + catch (EndOfStreamException) + { + // Catch this but don't count it as an error because SharpCompress is unsafe + } + catch (InvalidOperationException) + { + encounteredErrors = true; + } + catch (Exception) + { + // Don't log file open errors + encounteredErrors = true; + } + + return encounteredErrors; + } + + /// + /// Attempt to extract a file from an archive + /// + /// Name of the entry to be extracted + /// Output directory for archive extraction + /// Name of the extracted file, null on error + public override string ExtractEntry(string entryName, string outDir) + { + // Try to extract a stream using the given information + (MemoryStream ms, string realEntry) = ExtractEntryStream(entryName); + + // If the memory stream and the entry name are both non-null, we write to file + if (ms != null && realEntry != null) + { + realEntry = Path.Combine(outDir, realEntry); + + // Create the output subfolder now + Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); + + // Now open and write the file if possible + FileStream fs = FileTools.TryCreate(realEntry); + if (fs != null) + { + ms.Seek(0, SeekOrigin.Begin); + byte[] zbuffer = new byte[_bufferSize]; + int zlen; + while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) + { + fs.Write(zbuffer, 0, zlen); + fs.Flush(); + } + + ms?.Dispose(); + fs?.Dispose(); + } + else + { + ms?.Dispose(); + fs?.Dispose(); + realEntry = null; + } + } + + return realEntry; + } + + /// + /// Attempt to extract a stream from an archive + /// + /// Name of the entry to be extracted + /// Output representing the entry name that was found + /// MemoryStream representing the entry, null on error + public override (MemoryStream, string) ExtractEntryStream(string entryName) + { + MemoryStream ms = new MemoryStream(); + string realEntry = null; + + try + { + ZipFile zf = new ZipFile(); + ZipReturn zr = zf.Open(_filename, new FileInfo(_filename).LastWriteTime.Ticks, true); + if (zr != ZipReturn.ZipGood) + { + throw new Exception(ZipFile.ZipErrorMessageText(zr)); + } + + for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) + { + if (zf.Entries[i].FileName.Contains(entryName)) + { + // Open the read stream + realEntry = zf.Entries[i].FileName; + zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); + + // If the stream is smaller than the buffer, just run one loop through to avoid issues + if (streamsize < _bufferSize) + { + byte[] ibuffer = new byte[streamsize]; + int ilen = readStream.Read(ibuffer, 0, (int)streamsize); + ms.Write(ibuffer, 0, ilen); + ms.Flush(); + } + // Otherwise, we do the normal loop + else + { + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while (streamsize > _bufferSize) + { + ilen = readStream.Read(ibuffer, 0, _bufferSize); + ms.Write(ibuffer, 0, ilen); + ms.Flush(); + streamsize -= _bufferSize; + } + + ilen = readStream.Read(ibuffer, 0, (int)streamsize); + ms.Write(ibuffer, 0, ilen); + ms.Flush(); + } + + zr = zf.CloseReadStream(); + } + } + + zf.Dispose(); + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + ms = null; + realEntry = null; + } + + return (ms, realEntry); + } + + #endregion + + #region Information + + /// + /// Generate a list of DatItem objects from the header values in an archive + /// + /// Hash representing the hashes that should be skipped + /// True if entry dates should be included, false otherwise (default) + /// List of DatItem objects representing the found data + /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually + public override List GetArchiveFileInfo(Hash omitFromScan = Hash.DeepHashes, bool date = false) + { + List found = new List(); + string gamename = Path.GetFileNameWithoutExtension(_filename); + + try + { + ZipFile zf = new ZipFile(); + ZipReturn zr = zf.Open(_filename, new FileInfo(_filename).LastWriteTime.Ticks, true); + if (zr != ZipReturn.ZipGood) + { + throw new Exception(ZipFile.ZipErrorMessageText(zr)); + } + + for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) + { + // Open the read stream + zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); + + // If the entry ends with a directory separator, continue to the next item, if any + if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) + || zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) + || zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString())) + { + continue; + } + + // If secure hashes are disabled, do a quickscan + if (omitFromScan == Hash.SecureHashes) + { + string newname = zf.Entries[i].FileName; + long newsize = (long)zf.Entries[i].UncompressedSize; + string newcrc = BitConverter.ToString(zf.Entries[i].CRC.Reverse().ToArray(), 0, zf.Entries[i].CRC.Length).Replace("-", string.Empty).ToLowerInvariant(); + string convertedDate = Style.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss"); + + found.Add(new Rom + { + Type = ItemType.Rom, + Name = newname, + Size = newsize, + CRC = newcrc, + Date = (date ? convertedDate : null), + + MachineName = gamename, + }); + } + // Otherwise, use the stream directly + else + { + Rom zipEntryRom = (Rom)FileTools.GetStreamInfo(readStream, (long)zf.Entries[i].UncompressedSize, omitFromScan: omitFromScan); + zipEntryRom.Name = zf.Entries[i].FileName; + zipEntryRom.MachineName = gamename; + string convertedDate = Style.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss"); + zipEntryRom.Date = (date ? convertedDate : null); + found.Add(zipEntryRom); + zr = zf.CloseReadStream(); + } + } + + + // Dispose of the archive + zf.Close(); + } + catch (Exception) + { + // Don't log file open errors + return null; + } + + return found; + } + + /// + /// Generate a list of empty folders in an archive + /// + /// Input file to get data from + /// List of empty folders in the archive + public override List GetEmptyFolders() + { + List empties = new List(); + + try + { + SharpCompress.Archives.Zip.ZipArchive za = SharpCompress.Archives.Zip.ZipArchive.Open(_filename, new ReaderOptions { LeaveStreamOpen = false }); + List zipEntries = za.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); + string lastZipEntry = null; + foreach (SharpCompress.Archives.Zip.ZipArchiveEntry entry in zipEntries) + { + if (entry != null) + { + // If the current is a superset of last, we skip it + if (lastZipEntry != null && lastZipEntry.StartsWith(entry.Key)) + { + // No-op + } + // If the entry is a directory, we add it + else + { + if (entry.IsDirectory) + { + empties.Add(entry.Key); + } + lastZipEntry = entry.Key; + } + } + } + } + catch (Exception ex) + { + Globals.Logger.Error(ex.ToString()); + } + + return empties; + } + + #endregion + + #region Writing + + /// + /// Write an input file to a torrentzip archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(string inputFile, string outDir, Rom rom, bool date = false, bool romba = false) + { + // Get the file stream for the file and write out + return Write(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); + } + + /// + /// Write an input stream to a torrentzip archive + /// + /// Input filename to be moved + /// Output directory to build to + /// DatItem representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either input is null or empty, return + if (inputStream == null || rom == null || rom.Name == null) + { + return success; + } + + // If the stream is not readable, return + if (!inputStream.CanRead) + { + return success; + } + + // Seek to the beginning of the stream + inputStream.Seek(0, SeekOrigin.Begin); + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".zip") ? "" : ".zip")); + + // Set internal variables + Stream writeStream = null; + ZipFile oldZipFile = new ZipFile(); + ZipFile zipFile = new ZipFile(); + ZipReturn zipReturn = ZipReturn.ZipGood; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + inputStream.Seek(0, SeekOrigin.Begin); + zipReturn = zipFile.Create(tempFile); + + // Open the input file for reading + ulong istreamSize = (ulong)(inputStream.Length); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt)) + { + uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); + zipFile.OpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, + SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); + } + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + inputStream.Dispose(); + zipFile.CloseWriteStream(Convert.ToUInt32(rom.CRC, 16)); + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + oldZipFile.Open(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + + // If the old one doesn't contain the new file, then add it + if (!oldZipFile.Contains(rom.Name.Replace('\\', '/'))) + { + inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.EntriesCount; i++) + { + inputIndexMap.Add(oldZipFile.Filename(i), i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.EntriesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile.Create(tempFile); + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + // Open the input file for reading + ulong istreamSize = (ulong)(inputStream.Length); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt)) + { + uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); + zipFile.OpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, + SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); + } + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + inputStream.Dispose(); + zipFile.CloseWriteStream(Convert.ToUInt32(rom.CRC, 16)); + } + + // Otherwise, copy the file from the old archive + else + { + // Instantiate the streams + oldZipFile.OpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out SabreTools.Library.Data.CompressionMethod icompressionMethod, out uint lastMod); + zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index), + istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: lastMod); + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + zipFile.CloseWriteStream(BitConverter.ToUInt32(oldZipFile.CRC32(index), 0)); + } + } + } + + // Close the output zip file + zipFile.Close(); + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + inputStream?.Dispose(); + zipFile.Dispose(); + oldZipFile.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + return true; + } + + /// + /// Write a set of input files to a torrentzip archive (assuming the same output archive name) + /// + /// Input filenames to be moved + /// Output directory to build to + /// List of Rom representing the new information + /// True if the date from the DAT should be used if available, false otherwise (default) + /// True if files should be output in Romba depot folders, false otherwise + /// True if the archive was written properly, false otherwise + public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) + { + bool success = false; + string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); + + // If either list of roms is null or empty, return + if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) + { + return success; + } + + // If the number of inputs is less than the number of available roms, return + if (inputFiles.Count < roms.Count) + { + return success; + } + + // If one of the files doesn't exist, return + foreach (string file in inputFiles) + { + if (!File.Exists(file)) + { + return success; + } + } + + // Get the output archive name from the first rebuild rom + string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".zip") ? "" : ".zip")); + + // Set internal variables + Stream writeStream = null; + ZipFile oldZipFile = new ZipFile(); + ZipFile zipFile = new ZipFile(); + ZipReturn zipReturn = ZipReturn.ZipGood; + + try + { + // If the full output path doesn't exist, create it + if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) + { + Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); + } + + // If the archive doesn't exist, create it and put the single file + if (!File.Exists(archiveFileName)) + { + zipReturn = zipFile.Create(tempFile); + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); + } + + // Sort the keys in TZIP order + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Now add all of the files in order + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // Open the input file for reading + Stream freadStream = FileTools.TryOpenRead(inputFiles[index]); + ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt)) + { + uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); + zipFile.OpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize, + SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); + } + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + freadStream.Dispose(); + zipFile.CloseWriteStream(Convert.ToUInt32(roms[index].CRC, 16)); + } + } + + // Otherwise, sort the input files and write out in the correct order + else + { + // Open the old archive for reading + oldZipFile.Open(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); + + // Map all inputs to index + Dictionary inputIndexMap = new Dictionary(); + for (int i = 0; i < inputFiles.Count; i++) + { + // If the old one contains the new file, then just skip out + if (oldZipFile.Contains(roms[i].Name.Replace('\\', '/'))) + { + continue; + } + + inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); + } + + // Then add all of the old entries to it too + for (int i = 0; i < oldZipFile.EntriesCount; i++) + { + inputIndexMap.Add(oldZipFile.Filename(i), i); + } + + // If the number of entries is the same as the old archive, skip out + if (inputIndexMap.Keys.Count <= oldZipFile.EntriesCount) + { + success = true; + return success; + } + + // Otherwise, process the old zipfile + zipFile.Create(tempFile); + + // Get the order for the entries with the new file + List keys = inputIndexMap.Keys.ToList(); + keys.Sort(ZipFile.TorrentZipStringCompare); + + // Copy over all files to the new archive + foreach (string key in keys) + { + // Get the index mapped to the key + int index = inputIndexMap[key]; + + // If we have the input file, add it now + if (index < 0) + { + // Open the input file for reading + Stream freadStream = FileTools.TryOpenRead(inputFiles[-index - 1]); + ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt)) + { + uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); + zipFile.OpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, + SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); + } + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + freadStream.Dispose(); + zipFile.CloseWriteStream(Convert.ToUInt32(roms[-index - 1].CRC, 16)); + } + + // Otherwise, copy the file from the old archive + else + { + // Instantiate the streams + oldZipFile.OpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out SabreTools.Library.Data.CompressionMethod icompressionMethod, out uint lastMod); + zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index), + istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: lastMod); + + // Copy the input stream to the output + byte[] ibuffer = new byte[_bufferSize]; + int ilen; + while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0) + { + writeStream.Write(ibuffer, 0, ilen); + writeStream.Flush(); + } + zipFile.CloseWriteStream(BitConverter.ToUInt32(oldZipFile.CRC32(index), 0)); + } + } + } + + // Close the output zip file + zipFile.Close(); + + success = true; + } + catch (Exception ex) + { + Console.WriteLine(ex); + success = false; + } + finally + { + zipFile.Dispose(); + oldZipFile.Dispose(); + } + + // If the old file exists, delete it and replace + if (File.Exists(archiveFileName)) + { + FileTools.TryDeleteFile(archiveFileName); + } + File.Move(tempFile, archiveFileName); + + return true; + } + + #endregion + } +} diff --git a/SabreTools.Library/README.1ST b/SabreTools.Library/README.1ST index 4dfe4e7a..d62b9894 100644 --- a/SabreTools.Library/README.1ST +++ b/SabreTools.Library/README.1ST @@ -555,7 +555,7 @@ Options: format but with custom header information. This is currently unused by any major application. - -txz Enable Torrent XZ output + -txz Enable Torrent XZ output [UNSUPPORTED] Instead of outputting files to folder, files will be rebuilt to Torrent XZ (TXZ) files. This format is based on the LZMA container format XZ, but with custom header information. This is currently @@ -703,7 +703,7 @@ Options: format but with custom header information. This is currently unused by any major application. - -txz Enable Torrent XZ output + -txz Enable Torrent XZ output [UNSUPPORTED] Instead of outputting files to folder, files will be rebuilt to Torrent XZ (TXZ) files. This format is based on the LZMA container format XZ, but with custom header information. This is currently diff --git a/SabreTools.Library/SabreTools.Library.csproj b/SabreTools.Library/SabreTools.Library.csproj index 54fff118..24e3707d 100644 --- a/SabreTools.Library/SabreTools.Library.csproj +++ b/SabreTools.Library/SabreTools.Library.csproj @@ -130,7 +130,7 @@ - + @@ -150,6 +150,15 @@ + + + + + + + + + diff --git a/SabreTools.Library/Tools/ArchiveTools.cs b/SabreTools.Library/Tools/ArchiveTools.cs index d3af5567..31420ecb 100644 --- a/SabreTools.Library/Tools/ArchiveTools.cs +++ b/SabreTools.Library/Tools/ArchiveTools.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text.RegularExpressions; using SabreTools.Library.Data; +using SabreTools.Library.FileTypes; using SabreTools.Library.Items; #if MONO @@ -12,23 +12,12 @@ using System.IO; using Alphaleonis.Win32.Filesystem; using BinaryReader = System.IO.BinaryReader; -using BinaryWriter = System.IO.BinaryWriter; -using EndOfStreamException = System.IO.EndOfStreamException; using FileStream = System.IO.FileStream; using MemoryStream = System.IO.MemoryStream; using SeekOrigin = System.IO.SeekOrigin; using Stream = System.IO.Stream; #endif -using Ionic.Zlib; -using ROMVault2.SupportedFiles.Zip; -using SevenZip; -using SharpCompress.Archives; -using SharpCompress.Archives.Rar; -using SharpCompress.Archives.SevenZip; -using SharpCompress.Archives.Tar; using SharpCompress.Common; -using SharpCompress.Readers; -using SharpCompress.Writers; namespace SabreTools.Library.Tools { @@ -37,28 +26,22 @@ namespace SabreTools.Library.Tools /// /// /// TODO: Full archive support for: RAR, LRZip, ZPAQ?, Zstd?, LZ4? - /// Torrent 7-zip: https://sourceforge.net/p/t7z/code/HEAD/tree/ - /// LRZIP: https://github.com/ckolivas/lrzip /// ZPAQ: https://github.com/zpaq/zpaq - In progress as external DLL /// Zstd: https://github.com/skbkontur/ZstdNet /// LZ4: https://github.com/lz4/lz4 /// public static class ArchiveTools { - private const int _bufferSize = 4096 * 128; - - #region Extraction + #region Factory /// - /// Attempt to extract a file as an archive + /// Create an archive of the specified type, if possible /// - /// Name of the file to be extracted - /// Output directory for archive extraction - /// ArchiveScanLevel representing the archive handling levels - /// True if the extraction was a success, false otherwise - public static bool ExtractArchive(string input, string outDir, ArchiveScanLevel archiveScanLevel) + /// Name of the file to create the archive from + /// Archive object representing the inputs + public static BaseArchive CreateArchiveFromExistingInput(string input) { - bool encounteredErrors = true; + BaseArchive archive = null; // First get the archive type ArchiveType? at = GetCurrentArchiveType(input); @@ -66,767 +49,91 @@ namespace SabreTools.Library.Tools // If we got back null, then it's not an archive, so we we return if (at == null) { - return encounteredErrors; + return archive; } - try + // Create the archive based on the type + Globals.Logger.Verbose("Found archive of type: {0}", at); + switch (at) { - // 7-zip - if (at == ArchiveType.SevenZip && (archiveScanLevel & ArchiveScanLevel.SevenZipInternal) != 0) - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - // Create the temp directory - Directory.CreateDirectory(outDir); - - // Extract all files to the temp directory - SevenZipArchive sza = SevenZipArchive.Open(FileTools.TryOpenRead(input)); - foreach (SevenZipArchiveEntry entry in sza.Entries) - { - entry.WriteToDirectory(outDir, new ExtractionOptions{ PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); - } - encounteredErrors = false; - sza.Dispose(); - } - - // GZip - else if (at == ArchiveType.GZip && (archiveScanLevel & ArchiveScanLevel.GZipInternal) != 0) - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - // Create the temp directory - Directory.CreateDirectory(outDir); - - // Decompress the input stream - FileStream outstream = FileTools.TryCreate(Path.Combine(outDir, Path.GetFileNameWithoutExtension(input))); - GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(input), Ionic.Zlib.CompressionMode.Decompress); - gzstream.CopyTo(outstream); - - // Dispose of the streams - outstream.Dispose(); - gzstream.Dispose(); - - encounteredErrors = false; - } - - // RAR - else if (at == ArchiveType.Rar && (archiveScanLevel & ArchiveScanLevel.RarInternal) != 0) - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - // Create the temp directory - Directory.CreateDirectory(outDir); - - // Extract all files to the temp directory - RarArchive ra = RarArchive.Open(input); - foreach (RarArchiveEntry entry in ra.Entries) - { - entry.WriteToDirectory(outDir, new ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); - } - encounteredErrors = false; - ra.Dispose(); - } - - // TAR - else if (at == ArchiveType.Tar && (archiveScanLevel & ArchiveScanLevel.TarInternal) != 0) - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - // Create the temp directory - Directory.CreateDirectory(outDir); - - // Extract all files to the temp directory - TarArchive ta = TarArchive.Open(input); - foreach (TarArchiveEntry entry in ta.Entries) - { - entry.WriteToDirectory(outDir, new ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); - } - encounteredErrors = false; - ta.Dispose(); - } - - // Zip - else if (at == ArchiveType.Zip && (archiveScanLevel & ArchiveScanLevel.ZipInternal) != 0) - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - // Create the temp directory - Directory.CreateDirectory(outDir); - - // Extract all files to the temp directory - ZipFile zf = new ZipFile(); - ZipReturn zr = zf.Open(input, new FileInfo(input).LastWriteTime.Ticks, true); - if (zr != ZipReturn.ZipGood) - { - throw new Exception(ZipFile.ZipErrorMessageText(zr)); - } - - for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) - { - // Open the read stream - zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); - - // Create the rest of the path, if needed - if (!String.IsNullOrEmpty(Path.GetDirectoryName(zf.Entries[i].FileName))) - { - Directory.CreateDirectory(Path.Combine(outDir, Path.GetDirectoryName(zf.Entries[i].FileName))); - } - - // If the entry ends with a directory separator, continue to the next item, if any - if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) - || zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) - || zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString())) - { - continue; - } - - FileStream writeStream = FileTools.TryCreate(Path.Combine(outDir, zf.Entries[i].FileName)); - - // If the stream is smaller than the buffer, just run one loop through to avoid issues - if (streamsize < _bufferSize) - { - byte[] ibuffer = new byte[streamsize]; - int ilen = readStream.Read(ibuffer, 0, (int)streamsize); - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - // Otherwise, we do the normal loop - else - { - int realBufferSize = (streamsize < _bufferSize ? (int)streamsize : _bufferSize); - byte[] ibuffer = new byte[realBufferSize]; - int ilen; - while ((ilen = readStream.Read(ibuffer, 0, realBufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - } - - zr = zf.CloseReadStream(); - writeStream.Dispose(); - } - zf.Close(); - encounteredErrors = false; - } - } - catch (EndOfStreamException) - { - // Catch this but don't count it as an error because SharpCompress is unsafe - } - catch (InvalidOperationException) - { - encounteredErrors = true; - } - catch (Exception) - { - // Don't log file open errors - encounteredErrors = true; + case ArchiveType.GZip: + archive = new TorrentGZipArchive(input); + break; + case ArchiveType.Rar: + archive = new TorrentRARArchive(input); + break; + case ArchiveType.SevenZip: + archive = new TorrentSevenZipArchive(input); + break; + case ArchiveType.Tar: + archive = new TorrentTarArchive(input); + break; + case ArchiveType.Zip: + archive = new TorrentZipArchive(input); + break; } - return encounteredErrors; + return archive; } /// - /// Attempt to extract a file from an archive + /// Create an archive of the specified type, if possible /// - /// Name of the archive to be extracted - /// Name of the entry to be extracted - /// Output directory for archive extraction - /// Name of the extracted file, null on error - public static string ExtractItem(string input, string entryName, string outDir) + /// SharpCompress.Common.ArchiveType representing the archive to create + /// Archive object representing the inputs + public static BaseArchive CreateArchiveFromArchiveType(ArchiveType archiveType) { - // Try to extract a stream using the given information - (MemoryStream ms, string realEntry) = ExtractStream(input, entryName); - - // If the memory stream and the entry name are both non-null, we write to file - if (ms != null && realEntry != null) + switch(archiveType) { - realEntry = Path.Combine(outDir, realEntry); - - // Create the output subfolder now - Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); - - // Now open and write the file if possible - FileStream fs = FileTools.TryCreate(realEntry); - if (fs != null) - { - ms.Seek(0, SeekOrigin.Begin); - byte[] zbuffer = new byte[_bufferSize]; - int zlen; - while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) - { - fs.Write(zbuffer, 0, zlen); - fs.Flush(); - } - - ms?.Dispose(); - fs?.Dispose(); - } - else - { - ms?.Dispose(); - fs?.Dispose(); - realEntry = null; - } + case ArchiveType.GZip: + return new TorrentGZipArchive(); + case ArchiveType.Rar: + return new TorrentRARArchive(); + case ArchiveType.SevenZip: + return new TorrentSevenZipArchive(); + case ArchiveType.Tar: + return new TorrentTarArchive(); + case ArchiveType.Zip: + return new TorrentZipArchive(); + default: + return null; } - - return realEntry; } /// - /// Attempt to extract a stream from an archive + /// Create an archive of the specified type, if possible /// - /// Name of the archive to be extracted - /// Name of the entry to be extracted - /// Output representing the entry name that was found - /// MemoryStream representing the entry, null on error - public static (MemoryStream, string) ExtractStream(string input, string entryName) + /// SabreTools.Library.Data.SharpCompress.OutputFormat representing the archive to create + /// Archive object representing the inputs + public static BaseArchive CreateArchiveFromOutputFormat(OutputFormat outputFormat) { - MemoryStream ms = new MemoryStream(); - string realEntry = null; - - // First get the archive type - ArchiveType? at = GetCurrentArchiveType(input); - - // If we got back null, then it's not an archive, so we we return - if (at == null) + switch (outputFormat) { - return (null, realEntry); + case OutputFormat.Folder: + return new Folder(); + case OutputFormat.TapeArchive: + return new TorrentTarArchive(); + case OutputFormat.Torrent7Zip: + return new TorrentSevenZipArchive(); + case OutputFormat.TorrentGzip: + return new TorrentGZipArchive(); + case OutputFormat.TorrentLrzip: + return new TorrentLRZArchive(); + case OutputFormat.TorrentRar: + return new TorrentRARArchive(); + case OutputFormat.TorrentXZ: + return new TorrentXZArchive(); + case OutputFormat.TorrentZip: + return new TorrentZipArchive(); + default: + return null; } - - try - { - switch (at) - { - case ArchiveType.SevenZip: - SevenZipArchive sza = SevenZipArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false, }); - foreach (SevenZipArchiveEntry entry in sza.Entries) - { - if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) - { - // Write the file out - realEntry = entry.Key; - entry.WriteTo(ms); - break; - } - } - sza.Dispose(); - break; - - case ArchiveType.GZip: - // Decompress the input stream - realEntry = Path.GetFileNameWithoutExtension(input); - GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(input), Ionic.Zlib.CompressionMode.Decompress); - - // Write the file out - byte[] gbuffer = new byte[_bufferSize]; - int glen; - while ((glen = gzstream.Read(gbuffer, 0, _bufferSize)) > 0) - { - - ms.Write(gbuffer, 0, glen); - ms.Flush(); - } - - // Dispose of the streams - gzstream.Dispose(); - break; - - case ArchiveType.Rar: - RarArchive ra = RarArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false, }); - foreach (RarArchiveEntry entry in ra.Entries) - { - if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) - { - // Write the file out - realEntry = entry.Key; - entry.WriteTo(ms); - } - } - ra.Dispose(); - break; - - case ArchiveType.Tar: - TarArchive ta = TarArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false, }); - foreach (TarArchiveEntry entry in ta.Entries) - { - if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) - { - // Write the file out - realEntry = entry.Key; - entry.WriteTo(ms); - } - } - ta.Dispose(); - break; - - case ArchiveType.Zip: - ZipFile zf = new ZipFile(); - ZipReturn zr = zf.Open(input, new FileInfo(input).LastWriteTime.Ticks, true); - if (zr != ZipReturn.ZipGood) - { - throw new Exception(ZipFile.ZipErrorMessageText(zr)); - } - - for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) - { - if (zf.Entries[i].FileName.Contains(entryName)) - { - // Open the read stream - realEntry = zf.Entries[i].FileName; - zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); - - // If the stream is smaller than the buffer, just run one loop through to avoid issues - if (streamsize < _bufferSize) - { - byte[] ibuffer = new byte[streamsize]; - int ilen = readStream.Read(ibuffer, 0, (int)streamsize); - ms.Write(ibuffer, 0, ilen); - ms.Flush(); - } - // Otherwise, we do the normal loop - else - { - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while (streamsize > _bufferSize) - { - ilen = readStream.Read(ibuffer, 0, _bufferSize); - ms.Write(ibuffer, 0, ilen); - ms.Flush(); - streamsize -= _bufferSize; - } - - ilen = readStream.Read(ibuffer, 0, (int)streamsize); - ms.Write(ibuffer, 0, ilen); - ms.Flush(); - } - - zr = zf.CloseReadStream(); - } - } - - zf.Dispose(); - break; - } - } - catch (Exception ex) - { - Globals.Logger.Error(ex.ToString()); - ms = null; - realEntry = null; - } - - return (ms, realEntry); } #endregion #region Information - /// - /// Generate a list of DatItem objects from the header values in an archive - /// - /// Input file to get data from - /// True if entry dates should be included, false otherwise (default) - /// List of DatItem objects representing the found data - public static List GetArchiveFileInfo(string input, bool date = false) - { - return GetExtendedArchiveFileInfo(input, Hash.SecureHashes, date: date); - } - - /// - /// Generate a list of empty folders in an archive - /// - /// Input file to get data from - /// List of empty folders in the archive - public static List GetEmptyFoldersInArchive(string input) - { - List empties = new List(); - string gamename = Path.GetFileNameWithoutExtension(input); - - // First, we check that there is anything being passed as the input - if (String.IsNullOrEmpty(input)) - { - return empties; - } - - // Next, get the archive type - ArchiveType? at = GetCurrentArchiveType(input); - - // If we got back null, then it's not an archive, so we we return - if (at == null) - { - return empties; - } - - IReader reader = null; - try - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - switch (at) - { - case ArchiveType.SevenZip: - SevenZipArchive sza = SevenZipArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false }); - List sevenZipEntries = sza.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); - string lastSevenZipEntry = null; - foreach (SevenZipArchiveEntry entry in sevenZipEntries) - { - if (entry != null) - { - // If the current is a superset of last, we skip it - if (lastSevenZipEntry != null && lastSevenZipEntry.StartsWith(entry.Key)) - { - // No-op - } - // If the entry is a directory, we add it - else if (entry.IsDirectory) - { - empties.Add(entry.Key); - lastSevenZipEntry = entry.Key; - } - } - } - break; - - case ArchiveType.GZip: - // GZip files don't contain directories - break; - - case ArchiveType.Rar: - RarArchive ra = RarArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false }); - List rarEntries = ra.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); - string lastRarEntry = null; - foreach (RarArchiveEntry entry in rarEntries) - { - if (entry != null) - { - // If the current is a superset of last, we skip it - if (lastRarEntry != null && lastRarEntry.StartsWith(entry.Key)) - { - // No-op - } - // If the entry is a directory, we add it - else if (entry.IsDirectory) - { - empties.Add(entry.Key); - lastRarEntry = entry.Key; - } - } - } - break; - - case ArchiveType.Tar: - TarArchive ta = TarArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false }); - List tarEntries = ta.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); - string lastTarEntry = null; - foreach (TarArchiveEntry entry in tarEntries) - { - if (entry != null) - { - // If the current is a superset of last, we skip it - if (lastTarEntry != null && lastTarEntry.StartsWith(entry.Key)) - { - // No-op - } - // If the entry is a directory, we add it - else if (entry.IsDirectory) - { - empties.Add(entry.Key); - lastTarEntry = entry.Key; - } - } - } - break; - - case ArchiveType.Zip: - SharpCompress.Archives.Zip.ZipArchive za = SharpCompress.Archives.Zip.ZipArchive.Open(input, new ReaderOptions { LeaveStreamOpen = false }); - List zipEntries = za.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); - string lastZipEntry = null; - foreach (SharpCompress.Archives.Zip.ZipArchiveEntry entry in zipEntries) - { - if (entry != null) - { - // If the current is a superset of last, we skip it - if (lastZipEntry != null && lastZipEntry.StartsWith(entry.Key)) - { - // No-op - } - // If the entry is a directory, we add it - else - { - if (entry.IsDirectory) - { - empties.Add(entry.Key); - } - lastZipEntry = entry.Key; - } - } - } - break; - } - } - catch (Exception ex) - { - Globals.Logger.Error(ex.ToString()); - } - finally - { - reader?.Dispose(); - } - - return empties; - } - - /// - /// Generate a list of DatItem objects from the header values in an archive - /// - /// Input file to get data from - /// Hash representing the hashes that should be skipped - /// True if entry dates should be included, false otherwise (default) - /// List of DatItem objects representing the found data - /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually - public static List GetExtendedArchiveFileInfo(string input, Hash omitFromScan = Hash.DeepHashes, bool date = false) - { - List found = new List(); - string gamename = Path.GetFileNameWithoutExtension(input); - - // First get the archive type - ArchiveType? at = GetCurrentArchiveType(input); - - // If we got back null, then it's not an archive, so we we return - if (at == null) - { - return null; - } - - // If we got back GZip, try to get TGZ info first - else if (at == ArchiveType.GZip) - { - Rom possibleTgz = GetTorrentGZFileInfo(input); - - // If it was, then add it to the outputs and continue - if (possibleTgz != null && possibleTgz.Name != null) - { - found.Add(possibleTgz); - return found; - } - } - - try - { - Globals.Logger.Verbose("Found archive of type: {0}", at); - - switch (at) - { - // 7-zip - case ArchiveType.SevenZip: - SevenZipArchive sza = SevenZipArchive.Open(FileTools.TryOpenRead(input)); - foreach (SevenZipArchiveEntry entry in sza.Entries.Where(e => e != null && !e.IsDirectory)) - { - // If secure hashes are disabled, do a quickscan - if (omitFromScan == Hash.SecureHashes) - { - found.Add(new Rom - { - Type = ItemType.Rom, - Name = entry.Key, - Size = entry.Size, - CRC = entry.Crc.ToString("X").ToLowerInvariant(), - Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), - - MachineName = gamename, - }); - } - // Otherwise, use the stream directly - else - { - Stream entryStream = entry.OpenEntryStream(); - Rom sevenZipEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); - sevenZipEntryRom.Name = entry.Key; - sevenZipEntryRom.MachineName = gamename; - sevenZipEntryRom.Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null); - found.Add(sevenZipEntryRom); - entryStream.Dispose(); - } - } - - // Dispose of the archive - sza.Dispose(); - break; - - // GZip - case ArchiveType.GZip: - // If secure hashes are disabled, do a quickscan - if (omitFromScan == Hash.SecureHashes) - { - Rom tempRom = new Rom(gamename, gamename, omitFromScan); - BinaryReader br = new BinaryReader(FileTools.TryOpenRead(input)); - br.BaseStream.Seek(-8, SeekOrigin.End); - byte[] headercrc = br.ReadBytes(4); - tempRom.CRC = BitConverter.ToString(headercrc.Reverse().ToArray()).Replace("-", string.Empty).ToLowerInvariant(); - byte[] headersize = br.ReadBytes(4); - tempRom.Size = BitConverter.ToInt32(headersize.Reverse().ToArray(), 0); - br.Dispose(); - - found.Add(tempRom); - } - // Otherwise, use the stream directly - else - { - GZipStream gzstream = new GZipStream(FileTools.TryOpenRead(input), Ionic.Zlib.CompressionMode.Decompress); - Rom gzipEntryRom = (Rom)FileTools.GetStreamInfo(gzstream, gzstream.Length, omitFromScan: omitFromScan); - gzipEntryRom.Name = gzstream.FileName; - gzipEntryRom.MachineName = gamename; - gzipEntryRom.Date = (date && gzstream.LastModified != null ? gzstream.LastModified?.ToString("yyyy/MM/dd hh:mm:ss") : null); - found.Add(gzipEntryRom); - gzstream.Dispose(); - } - break; - - // RAR - case ArchiveType.Rar: - RarArchive ra = RarArchive.Open(FileTools.TryOpenRead(input)); - foreach (RarArchiveEntry entry in ra.Entries.Where(e => e != null && !e.IsDirectory)) - { - // If secure hashes are disabled, do a quickscan - if (omitFromScan == Hash.SecureHashes) - { - found.Add(new Rom - { - Type = ItemType.Rom, - Name = entry.Key, - Size = entry.Size, - CRC = entry.Crc.ToString("X").ToLowerInvariant(), - Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), - - MachineName = gamename, - }); - } - // Otherwise, use the stream directly - else - { - Stream entryStream = entry.OpenEntryStream(); - Rom rarEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); - rarEntryRom.Name = entry.Key; - rarEntryRom.MachineName = gamename; - rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); - found.Add(rarEntryRom); - entryStream.Dispose(); - } - } - - // Dispose of the archive - ra.Dispose(); - break; - - // TAR - case ArchiveType.Tar: - TarArchive ta = TarArchive.Open(FileTools.TryOpenRead(input)); - foreach (TarArchiveEntry entry in ta.Entries.Where(e => e != null && !e.IsDirectory)) - { - // If secure hashes are disabled, do a quickscan - if (omitFromScan == Hash.SecureHashes) - { - found.Add(new Rom - { - Type = ItemType.Rom, - Name = entry.Key, - Size = entry.Size, - CRC = entry.Crc.ToString("X").ToLowerInvariant(), - Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null), - - MachineName = gamename, - }); - } - // Otherwise, use the stream directly - else - { - Stream entryStream = entry.OpenEntryStream(); - Rom tarEntryRom = (Rom)FileTools.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); - tarEntryRom.Name = entry.Key; - tarEntryRom.MachineName = gamename; - tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); - found.Add(tarEntryRom); - entryStream.Dispose(); - } - } - - // Dispose of the archive - ta.Dispose(); - break; - - // Zip - case ArchiveType.Zip: - ZipFile zf = new ZipFile(); - ZipReturn zr = zf.Open(input, new FileInfo(input).LastWriteTime.Ticks, true); - if (zr != ZipReturn.ZipGood) - { - throw new Exception(ZipFile.ZipErrorMessageText(zr)); - } - - for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++) - { - // Open the read stream - zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod); - - // If the entry ends with a directory separator, continue to the next item, if any - if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) - || zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) - || zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString())) - { - continue; - } - - // If secure hashes are disabled, do a quickscan - if (omitFromScan == Hash.SecureHashes) - { - string newname = zf.Entries[i].FileName; - long newsize = (long)zf.Entries[i].UncompressedSize; - string newcrc = BitConverter.ToString(zf.Entries[i].CRC.Reverse().ToArray(), 0, zf.Entries[i].CRC.Length).Replace("-", string.Empty).ToLowerInvariant(); - string convertedDate = Style.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss"); - - found.Add(new Rom - { - Type = ItemType.Rom, - Name = newname, - Size = newsize, - CRC = newcrc, - Date = (date ? convertedDate : null), - - MachineName = gamename, - }); - } - // Otherwise, use the stream directly - else - { - Rom zipEntryRom = (Rom)FileTools.GetStreamInfo(readStream, (long)zf.Entries[i].UncompressedSize, omitFromScan: omitFromScan); - zipEntryRom.Name = zf.Entries[i].FileName; - zipEntryRom.MachineName = gamename; - string convertedDate = Style.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss"); - zipEntryRom.Date = (date ? convertedDate : null); - found.Add(zipEntryRom); - zr = zf.CloseReadStream(); - } - } - - - // Dispose of the archive - zf.Close(); - break; - } - } - catch (Exception) - { - // Don't log file open errors - return null; - } - - return found; - } - /// /// Retrieve file information for a single torrent GZ file /// @@ -1361,1826 +668,5 @@ namespace SabreTools.Library.Tools } #endregion - - #region Writing - - /// - /// Write an input stream to file - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if we should overwrite the file if it exists, false otherwise - /// True if the file was written properly, false otherwise - public static bool WriteFile(Stream inputStream, string outDir, DatItem datItem, bool date = false, bool overwrite = false) - { - bool success = false; - - // If either input is null or empty, return - if (inputStream == null || datItem == null || datItem.Name == null) - { - return success; - } - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Set internal variables - FileStream outputStream = null; - - // Get the output folder name from the first rebuild rom - string fileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(datItem.MachineName), Style.RemovePathUnsafeCharacters(datItem.Name)); - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(fileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(fileName)); - } - - // If the file exists and we're supposed to overwrite or the file doesn't exist at all - if ((File.Exists(fileName) && overwrite) || !File.Exists(fileName)) - { - outputStream = FileTools.TryCreate(fileName); - } - - // If the output stream isn't null - if (outputStream != null) - { - // Copy the input stream to the output - inputStream.Seek(0, SeekOrigin.Begin); - int bufferSize = 4096 * 128; - byte[] ibuffer = new byte[bufferSize]; - int ilen; - while ((ilen = inputStream.Read(ibuffer, 0, bufferSize)) > 0) - { - outputStream.Write(ibuffer, 0, ilen); - outputStream.Flush(); - } - outputStream.Dispose(); - - if (datItem.Type == ItemType.Rom) - { - if (date && !String.IsNullOrEmpty(((Rom)datItem).Date)) - { - File.SetCreationTime(fileName, DateTime.Parse(((Rom)datItem).Date)); - } - } - - success = true; - } - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - inputStream.Dispose(); - outputStream?.Dispose(); - } - - return success; - } - - /// - /// Write an input file to a tape archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTAR(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTAR(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input stream to a tape archive - /// - /// Input stream to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTAR(Stream inputStream, string outDir, Rom rom, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(Path.GetTempPath(), "tmp" + Guid.NewGuid().ToString()); - - // If either input is null or empty, return - if (inputStream == null || rom == null || rom.Name == null) - { - return success; - } - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".tar") ? "" : ".tar")); - - // Set internal variables - TarArchive oldTarFile = TarArchive.Create(); - TarArchive tarFile = TarArchive.Create(); - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - // Copy the input stream to the output - inputStream.Seek(0, SeekOrigin.Begin); - tarFile.AddEntry(rom.Name, inputStream); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - oldTarFile = TarArchive.Open(archiveFileName); - - // Get a list of all current entries - List entries = oldTarFile.Entries.Select(i => i.Key).ToList(); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - - // If the old one doesn't contain the new file, then add it - if (!entries.Contains(rom.Name.Replace('\\', '/'))) - { - inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); - } - - // Then add all of the old entries to it too - for (int i = 0; i < entries.Count; i++) - { - inputIndexMap.Add(entries[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= entries.Count) - { - success = true; - return success; - } - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Copy the input file to the output - inputStream.Seek(0, SeekOrigin.Begin); - tarFile.AddEntry(rom.Name, inputStream); - } - - // Otherwise, copy the file from the old archive - else - { - // Get the stream from the original archive - string tempEntry = Path.Combine(Path.GetTempPath(), "tmp" + Guid.NewGuid().ToString()); - oldTarFile.Entries.Where(e => e.Key == key).ToList()[0].WriteToFile(tempEntry); - - // Copy the input stream to the output - tarFile.AddEntry(key, tempEntry); - } - } - } - - // Close the output tar file - tarFile.SaveTo(tempFile, new WriterOptions(CompressionType.None)); - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - inputStream.Dispose(); - tarFile.Dispose(); - oldTarFile.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - return success; - } - - /// - /// Write a set of input files to a tape archive (assuming the same output archive name) - /// - /// Input filenames to be moved - /// Output directory to build to - /// List of Rom representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTAR(List inputFiles, string outDir, List roms, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(Path.GetTempPath(), "tmp" + Guid.NewGuid().ToString()); - - // If either list of roms is null or empty, return - if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) - { - return success; - } - - // If the number of inputs is less than the number of available roms, return - if (inputFiles.Count < roms.Count) - { - return success; - } - - // If one of the files doesn't exist, return - foreach (string file in inputFiles) - { - if (!File.Exists(file)) - { - return success; - } - } - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".tar") ? "" : ".tar")); - - // Set internal variables - TarArchive oldTarFile = TarArchive.Create(); - TarArchive tarFile = TarArchive.Create(); - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); - } - - // Sort the keys in TZIP order - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Now add all of the files in order - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // Copy the input stream to the output - tarFile.AddEntry(roms[index].Name, inputFiles[index]); - } - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - oldTarFile = TarArchive.Open(archiveFileName); - - // Get a list of all current entries - List entries = oldTarFile.Entries.Select(i => i.Key).ToList(); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - // If the old one contains the new file, then just skip out - if (entries.Contains(roms[i].Name.Replace('\\', '/'))) - { - continue; - } - - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); - } - - // Then add all of the old entries to it too - for (int i = 0; i < entries.Count; i++) - { - inputIndexMap.Add(entries[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= entries.Count) - { - success = true; - return success; - } - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Copy the input file to the output - tarFile.AddEntry(roms[-index - 1].Name, inputFiles[-index - 1]); - } - - // Otherwise, copy the file from the old archive - else - { - // Get the stream from the original archive - string tempEntry = Path.Combine(Path.GetTempPath(), "tmp" + Guid.NewGuid().ToString()); - oldTarFile.Entries.Where(e => e.Key == key).ToList()[0].WriteToFile(tempEntry); - - // Copy the input stream to the output - tarFile.AddEntry(key, tempEntry); - } - } - } - - // Close the output tar file - tarFile.SaveTo(tempFile, new WriterOptions(CompressionType.None)); - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - tarFile.Dispose(); - oldTarFile.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - return true; - } - - /// - /// Write an input file to a torrent7z archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrent7Zip(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTorrent7Zip(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input file to a torrent7z archive - /// - /// Input stream to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrent7Zip(Stream inputStream, string outDir, Rom rom, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either input is null or empty, return - if (inputStream == null || rom == null || rom.Name == null) - { - return success; - } - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Seek to the beginning of the stream - inputStream.Seek(0, SeekOrigin.Begin); - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".7z") ? "" : ".7z")); - - // Set internal variables - SevenZipBase.SetLibraryPath("7za.dll"); - SevenZipExtractor oldZipFile; - SevenZipCompressor zipFile; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.SevenZip, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - - // Create the temp directory - string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); - if (!Directory.Exists(tempPath)) - { - Directory.CreateDirectory(tempPath); - } - - // Create a stream dictionary - Dictionary dict = new Dictionary(); - dict.Add(rom.Name, inputStream); - - // Now add the stream - zipFile.CompressStreamDictionary(dict, archiveFileName); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - Stream oldZipFileStream = FileTools.TryOpenRead(archiveFileName); - oldZipFile = new SevenZipExtractor(oldZipFileStream); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - - // If the old one doesn't contain the new file, then add it - if (!oldZipFile.ArchiveFileNames.Contains(rom.Name.Replace('\\', '/'))) - { - inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.FilesCount; i++) - { - inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.SevenZip, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - Stream zipFileStream = FileTools.TryOpenWrite(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Create a stream dictionary - Dictionary dict = new Dictionary(); - dict.Add(rom.Name, inputStream); - - // Now add the stream - zipFile.CompressStreamDictionary(dict, archiveFileName); - } - - // Otherwise, copy the file from the old archive - else - { - Stream oldZipFileEntryStream = new MemoryStream(); - oldZipFile.ExtractFile(index, oldZipFileEntryStream); - zipFile.CompressFiles(zipFileStream, key); - oldZipFileEntryStream.Dispose(); - } - } - - zipFileStream.Dispose(); - oldZipFile.Dispose(); - } - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - inputStream?.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - // Now make the file T7Z - // TODO: Add ACTUAL T7Z compatible code - - BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); - bw.Seek(0, SeekOrigin.Begin); - bw.Write(Constants.Torrent7ZipHeader); - bw.Seek(0, SeekOrigin.End); - - oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName)); - - // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) - byte[] tempsig = Constants.Torrent7ZipSignature; - if (oldZipFile.FilesCount > 1) - { - tempsig[16] = 0x2; - } - else - { - tempsig[16] = 0; - } - - bw.Write(tempsig); - bw.Dispose(); - - return true; - } - - /// - /// Write a set of input files to a torrent7z archive (assuming the same output archive name) - /// - /// Input filenames to be moved - /// Output directory to build to - /// List of Rom representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrent7Zip(List inputFiles, string outDir, List roms, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either list of roms is null or empty, return - if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) - { - return success; - } - - // If the number of inputs is less than the number of available roms, return - if (inputFiles.Count < roms.Count) - { - return success; - } - - // If one of the files doesn't exist, return - foreach (string file in inputFiles) - { - if (!File.Exists(file)) - { - return success; - } - } - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".7z") ? "" : ".7z")); - - // Set internal variables - SevenZipBase.SetLibraryPath("7za.dll"); - SevenZipExtractor oldZipFile; - SevenZipCompressor zipFile; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.SevenZip, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); - } - - // Sort the keys in TZIP order - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Create the temp directory - string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); - if (!Directory.Exists(tempPath)) - { - Directory.CreateDirectory(tempPath); - } - - // Now add all of the files in order - foreach (string key in keys) - { - string newkey = Path.Combine(tempPath, key); - - File.Move(inputFiles[inputIndexMap[key]], newkey); - zipFile.CompressFiles(tempFile, newkey); - File.Move(newkey, inputFiles[inputIndexMap[key]]); - } - - FileTools.CleanDirectory(tempPath); - FileTools.TryDeleteDirectory(tempPath); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - Stream oldZipFileStream = FileTools.TryOpenRead(archiveFileName); - oldZipFile = new SevenZipExtractor(oldZipFileStream); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - // If the old one contains the new file, then just skip out - if (oldZipFile.ArchiveFileNames.Contains(roms[i].Name.Replace('\\', '/'))) - { - continue; - } - - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.FilesCount; i++) - { - inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.SevenZip, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - Stream zipFileStream = FileTools.TryOpenWrite(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - zipFile.CompressFiles(zipFileStream, inputFiles[-index - 1]); - } - - // Otherwise, copy the file from the old archive - else - { - Stream oldZipFileEntryStream = FileTools.TryOpenReadWrite(inputFiles[index]); - oldZipFile.ExtractFile(index, oldZipFileEntryStream); - zipFile.CompressFiles(zipFileStream, inputFiles[index]); - - oldZipFileEntryStream.Dispose(); - FileTools.TryDeleteFile(inputFiles[index]); - } - } - - zipFileStream.Dispose(); - oldZipFile.Dispose(); - } - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - // Now make the file T7Z - // TODO: Add ACTUAL T7Z compatible code - - BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); - bw.Seek(0, SeekOrigin.Begin); - bw.Write(Constants.Torrent7ZipHeader); - bw.Seek(0, SeekOrigin.End); - - oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName)); - - // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) - byte[] tempsig = Constants.Torrent7ZipSignature; - if (oldZipFile.FilesCount > 1) - { - tempsig[16] = 0x2; - } - else - { - tempsig[16] = 0; - } - - bw.Write(tempsig); - bw.Dispose(); - - return true; - } - - /// - /// Write an input file to a torrent GZ file - /// - /// File to write from - /// Directory to write archive to - /// True if files should be output in Romba depot folders, false otherwise - /// True if the write was a success, false otherwise - /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. - public static bool WriteTorrentGZ(string inputFile, string outDir, bool romba) - { - // Check that the input file exists - if (!File.Exists(inputFile)) - { - Globals.Logger.Warning("File '{0}' does not exist!", inputFile); - return false; - } - inputFile = Path.GetFullPath(inputFile); - - // Get the file stream for the file and write out - return WriteTorrentGZ(FileTools.TryOpenRead(inputFile), outDir, romba); - } - - /// - /// Write an input stream to a torrent GZ file - /// - /// File to write from - /// Directory to write archive to - /// True if files should be output in Romba depot folders, false otherwise - /// True if the write was a success, false otherwise - /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. - public static bool WriteTorrentGZ(Stream inputStream, string outDir, bool romba) - { - bool success = false; - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Make sure the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - } - outDir = Path.GetFullPath(outDir); - - // Now get the Rom info for the file so we have hashes and size - Rom rom = (Rom)FileTools.GetStreamInfo(inputStream, inputStream.Length, keepReadOpen: true); - - // Get the output file name - string outfile = null; - - // If we have a romba output, add the romba path - if (romba) - { - outfile = Path.Combine(outDir, Style.GetRombaPath(rom.SHA1)); // TODO: When updating to SHA-256, this needs to update to SHA256 - - // Check to see if the folder needs to be created - if (!Directory.Exists(Path.GetDirectoryName(outfile))) - { - Directory.CreateDirectory(Path.GetDirectoryName(outfile)); - } - } - // Otherwise, we're just rebuilding to the main directory - else - { - outfile = Path.Combine(outDir, rom.SHA1 + ".gz"); // TODO: When updating to SHA-256, this needs to update to SHA256 - } - - // If the output file exists, don't try to write again - if (!File.Exists(outfile)) - { - // Compress the input stream - FileStream outputStream = FileTools.TryCreate(outfile); - - // Open the output file for writing - BinaryWriter sw = new BinaryWriter(outputStream); - - // Write standard header and TGZ info - byte[] data = Constants.TorrentGZHeader - .Concat(Style.StringToByteArray(rom.MD5)) // MD5 - .Concat(Style.StringToByteArray(rom.CRC)) // CRC - .ToArray(); - sw.Write(data); - sw.Write((ulong)rom.Size); // Long size (Unsigned, Mirrored) - - // Now create a deflatestream from the input file - DeflateStream ds = new DeflateStream(outputStream, Ionic.Zlib.CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true); - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - ds.Write(ibuffer, 0, ilen); - ds.Flush(); - } - ds.Dispose(); - - // Now write the standard footer - sw.Write(Style.StringToByteArray(rom.CRC).Reverse().ToArray()); - sw.Write((uint)rom.Size); - - // Dispose of everything - sw.Dispose(); - outputStream.Dispose(); - inputStream.Dispose(); - } - - return true; - } - - /// - /// Write an input file to a torrentlrzip archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentLRZ(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTorrentLRZ(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input stream to a torrentlrzip archive - /// - /// Input stream to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentLRZ(Stream inputStream, string outDir, Rom rom, bool date = false) - { - throw new NotImplementedException(); - } - - /// - /// Write an input file to a torrentrar archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentRAR(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTorrentRAR(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input stream to a torrentrar archive - /// - /// Input stream to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentRAR(Stream inputStream, string outDir, Rom rom, bool date = false) - { - throw new NotImplementedException(); - } - - /// - /// Write an input file to a torrentxz archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentXZ(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTorrentXZ(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input file to a torrentxz archive - /// - /// Input stream to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentXZ(Stream inputStream, string outDir, Rom rom, bool date = false) - - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either input is null or empty, return - if (inputStream == null || rom == null || rom.Name == null) - { - return success; - } - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Seek to the beginning of the stream - inputStream.Seek(0, SeekOrigin.Begin); - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".xz") ? "" : ".xz")); - - // Set internal variables - SevenZipBase.SetLibraryPath("7za.dll"); - SevenZipExtractor oldZipFile; - SevenZipCompressor zipFile; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.XZ, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - - // Create the temp directory - string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); - if (!Directory.Exists(tempPath)) - { - Directory.CreateDirectory(tempPath); - } - - // Create a stream dictionary - Dictionary dict = new Dictionary(); - dict.Add(rom.Name, inputStream); - - // Now add the stream - zipFile.CompressStreamDictionary(dict, archiveFileName); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - Stream oldZipFileStream = FileTools.TryOpenRead(archiveFileName); - oldZipFile = new SevenZipExtractor(oldZipFileStream); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - - // If the old one doesn't contain the new file, then add it - if (!oldZipFile.ArchiveFileNames.Contains(rom.Name.Replace('\\', '/'))) - { - inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.FilesCount; i++) - { - inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.XZ, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - Stream zipFileStream = FileTools.TryOpenWrite(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Create a stream dictionary - Dictionary dict = new Dictionary(); - dict.Add(rom.Name, inputStream); - - // Now add the stream - zipFile.CompressStreamDictionary(dict, archiveFileName); - } - - // Otherwise, copy the file from the old archive - else - { - Stream oldZipFileEntryStream = new MemoryStream(); - oldZipFile.ExtractFile(index, oldZipFileEntryStream); - zipFile.CompressFiles(zipFileStream, key); - oldZipFileEntryStream.Dispose(); - } - } - - zipFileStream.Dispose(); - oldZipFile.Dispose(); - } - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - inputStream?.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - // Now make the file TXZ - // TODO: Add ACTUAL TXZ compatible code (based on T7z) - - BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); - bw.Seek(0, SeekOrigin.Begin); - bw.Write(Constants.Torrent7ZipHeader); - bw.Seek(0, SeekOrigin.End); - - oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName)); - - // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) - byte[] tempsig = Constants.Torrent7ZipSignature; - if (oldZipFile.FilesCount > 1) - { - tempsig[16] = 0x2; - } - else - { - tempsig[16] = 0; - } - - bw.Write(tempsig); - bw.Dispose(); - - return true; - } - - /// - /// Write a set of input files to a torrentxz archive (assuming the same output archive name) - /// - /// Input filenames to be moved - /// Output directory to build to - /// List of Rom representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentXZ(List inputFiles, string outDir, List roms, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either list of roms is null or empty, return - if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) - { - return success; - } - - // If the number of inputs is less than the number of available roms, return - if (inputFiles.Count < roms.Count) - { - return success; - } - - // If one of the files doesn't exist, return - foreach (string file in inputFiles) - { - if (!File.Exists(file)) - { - return success; - } - } - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".7z") ? "" : ".7z")); - - // Set internal variables - SevenZipBase.SetLibraryPath("7za.dll"); - SevenZipExtractor oldZipFile; - SevenZipCompressor zipFile; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.XZ, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); - } - - // Sort the keys in TZIP order - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Create the temp directory - string tempPath = Path.Combine(Path.GetTempPath(), new Guid().ToString()); - if (!Directory.Exists(tempPath)) - { - Directory.CreateDirectory(tempPath); - } - - // Now add all of the files in order - foreach (string key in keys) - { - string newkey = Path.Combine(tempPath, key); - - File.Move(inputFiles[inputIndexMap[key]], newkey); - zipFile.CompressFiles(tempFile, newkey); - File.Move(newkey, inputFiles[inputIndexMap[key]]); - } - - FileTools.CleanDirectory(tempPath); - FileTools.TryDeleteDirectory(tempPath); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - Stream oldZipFileStream = FileTools.TryOpenRead(archiveFileName); - oldZipFile = new SevenZipExtractor(oldZipFileStream); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - // If the old one contains the new file, then just skip out - if (oldZipFile.ArchiveFileNames.Contains(roms[i].Name.Replace('\\', '/'))) - { - continue; - } - - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.FilesCount; i++) - { - inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile = new SevenZipCompressor() - { - ArchiveFormat = OutArchiveFormat.XZ, - CompressionLevel = SevenZip.CompressionLevel.Normal, - }; - Stream zipFileStream = FileTools.TryOpenWrite(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - zipFile.CompressFiles(zipFileStream, inputFiles[-index - 1]); - } - - // Otherwise, copy the file from the old archive - else - { - Stream oldZipFileEntryStream = FileTools.TryCreate(inputFiles[index]); - oldZipFile.ExtractFile(index, oldZipFileEntryStream); - zipFile.CompressFiles(zipFileStream, inputFiles[index]); - - oldZipFileEntryStream.Dispose(); - FileTools.TryDeleteFile(inputFiles[index]); - } - } - - zipFileStream.Dispose(); - oldZipFile.Dispose(); - } - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - // Now make the file TXZ - // TODO: Add ACTUAL TXZ compatible code (based on T7z) - - BinaryWriter bw = new BinaryWriter(FileTools.TryOpenReadWrite(archiveFileName)); - bw.Seek(0, SeekOrigin.Begin); - bw.Write(Constants.Torrent7ZipHeader); - bw.Seek(0, SeekOrigin.End); - - oldZipFile = new SevenZipExtractor(FileTools.TryOpenReadWrite(archiveFileName)); - - // Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4) - byte[] tempsig = Constants.Torrent7ZipSignature; - if (oldZipFile.FilesCount > 1) - { - tempsig[16] = 0x2; - } - else - { - tempsig[16] = 0; - } - - bw.Write(tempsig); - bw.Dispose(); - - return true; - } - - /// - /// Write an input file to a torrentzip archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentZip(string inputFile, string outDir, Rom rom, bool date = false) - { - // Get the file stream for the file and write out - return WriteTorrentZip(FileTools.TryOpenRead(inputFile), outDir, rom, date: date); - } - - /// - /// Write an input stream to a torrentzip archive - /// - /// Input filename to be moved - /// Output directory to build to - /// DatItem representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentZip(Stream inputStream, string outDir, Rom rom, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either input is null or empty, return - if (inputStream == null || rom == null || rom.Name == null) - { - return success; - } - - // If the stream is not readable, return - if (!inputStream.CanRead) - { - return success; - } - - // Seek to the beginning of the stream - inputStream.Seek(0, SeekOrigin.Begin); - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".zip") ? "" : ".zip")); - - // Set internal variables - Stream writeStream = null; - ZipFile oldZipFile = new ZipFile(); - ZipFile zipFile = new ZipFile(); - ZipReturn zipReturn = ZipReturn.ZipGood; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - inputStream.Seek(0, SeekOrigin.Begin); - zipReturn = zipFile.Create(tempFile); - - // Open the input file for reading - ulong istreamSize = (ulong)(inputStream.Length); - - DateTime dt = DateTime.Now; - if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt)) - { - uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); - zipFile.OpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, - SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); - } - else - { - zipFile.OpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); - } - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - inputStream.Dispose(); - zipFile.CloseWriteStream(Convert.ToUInt32(rom.CRC, 16)); - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - oldZipFile.Open(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - - // If the old one doesn't contain the new file, then add it - if (!oldZipFile.Contains(rom.Name.Replace('\\', '/'))) - { - inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.EntriesCount; i++) - { - inputIndexMap.Add(oldZipFile.Filename(i), i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.EntriesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile.Create(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Open the input file for reading - ulong istreamSize = (ulong)(inputStream.Length); - - DateTime dt = DateTime.Now; - if (date && !String.IsNullOrEmpty(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt)) - { - uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); - zipFile.OpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, - SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); - } - else - { - zipFile.OpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); - } - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - inputStream.Dispose(); - zipFile.CloseWriteStream(Convert.ToUInt32(rom.CRC, 16)); - } - - // Otherwise, copy the file from the old archive - else - { - // Instantiate the streams - oldZipFile.OpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out SabreTools.Library.Data.CompressionMethod icompressionMethod, out uint lastMod); - zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index), - istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: lastMod); - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - zipFile.CloseWriteStream(BitConverter.ToUInt32(oldZipFile.CRC32(index), 0)); - } - } - } - - // Close the output zip file - zipFile.Close(); - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - inputStream?.Dispose(); - zipFile.Dispose(); - oldZipFile.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - return true; - } - - /// - /// Write a set of input files to a torrentzip archive (assuming the same output archive name) - /// - /// Input filenames to be moved - /// Output directory to build to - /// List of Rom representing the new information - /// True if the date from the DAT should be used if available, false otherwise (default) - /// True if the archive was written properly, false otherwise - public static bool WriteTorrentZip(List inputFiles, string outDir, List roms, bool date = false) - { - bool success = false; - string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString()); - - // If either list of roms is null or empty, return - if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0) - { - return success; - } - - // If the number of inputs is less than the number of available roms, return - if (inputFiles.Count < roms.Count) - { - return success; - } - - // If one of the files doesn't exist, return - foreach (string file in inputFiles) - { - if (!File.Exists(file)) - { - return success; - } - } - - // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".zip") ? "" : ".zip")); - - // Set internal variables - Stream writeStream = null; - ZipFile oldZipFile = new ZipFile(); - ZipFile zipFile = new ZipFile(); - ZipReturn zipReturn = ZipReturn.ZipGood; - - try - { - // If the full output path doesn't exist, create it - if (!Directory.Exists(Path.GetDirectoryName(archiveFileName))) - { - Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName)); - } - - // If the archive doesn't exist, create it and put the single file - if (!File.Exists(archiveFileName)) - { - zipReturn = zipFile.Create(tempFile); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i); - } - - // Sort the keys in TZIP order - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Now add all of the files in order - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // Open the input file for reading - Stream freadStream = FileTools.TryOpenRead(inputFiles[index]); - ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length); - - DateTime dt = DateTime.Now; - if (date && !String.IsNullOrEmpty(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt)) - { - uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); - zipFile.OpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize, - SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); - } - else - { - zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); - } - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - freadStream.Dispose(); - zipFile.CloseWriteStream(Convert.ToUInt32(roms[index].CRC, 16)); - } - } - - // Otherwise, sort the input files and write out in the correct order - else - { - // Open the old archive for reading - oldZipFile.Open(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); - - // Map all inputs to index - Dictionary inputIndexMap = new Dictionary(); - for (int i = 0; i < inputFiles.Count; i++) - { - // If the old one contains the new file, then just skip out - if (oldZipFile.Contains(roms[i].Name.Replace('\\', '/'))) - { - continue; - } - - inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1)); - } - - // Then add all of the old entries to it too - for (int i = 0; i < oldZipFile.EntriesCount; i++) - { - inputIndexMap.Add(oldZipFile.Filename(i), i); - } - - // If the number of entries is the same as the old archive, skip out - if (inputIndexMap.Keys.Count <= oldZipFile.EntriesCount) - { - success = true; - return success; - } - - // Otherwise, process the old zipfile - zipFile.Create(tempFile); - - // Get the order for the entries with the new file - List keys = inputIndexMap.Keys.ToList(); - keys.Sort(ZipFile.TorrentZipStringCompare); - - // Copy over all files to the new archive - foreach (string key in keys) - { - // Get the index mapped to the key - int index = inputIndexMap[key]; - - // If we have the input file, add it now - if (index < 0) - { - // Open the input file for reading - Stream freadStream = FileTools.TryOpenRead(inputFiles[-index - 1]); - ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length); - - DateTime dt = DateTime.Now; - if (date && !String.IsNullOrEmpty(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt)) - { - uint msDosDateTime = Style.ConvertDateTimeToMsDosTimeFormat(dt); - zipFile.OpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, - SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); - } - else - { - zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream); - } - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - freadStream.Dispose(); - zipFile.CloseWriteStream(Convert.ToUInt32(roms[-index - 1].CRC, 16)); - } - - // Otherwise, copy the file from the old archive - else - { - // Instantiate the streams - oldZipFile.OpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out SabreTools.Library.Data.CompressionMethod icompressionMethod, out uint lastMod); - zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index), - istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: lastMod); - - // Copy the input stream to the output - byte[] ibuffer = new byte[_bufferSize]; - int ilen; - while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0) - { - writeStream.Write(ibuffer, 0, ilen); - writeStream.Flush(); - } - zipFile.CloseWriteStream(BitConverter.ToUInt32(oldZipFile.CRC32(index), 0)); - } - } - } - - // Close the output zip file - zipFile.Close(); - - success = true; - } - catch (Exception ex) - { - Console.WriteLine(ex); - success = false; - } - finally - { - zipFile.Dispose(); - oldZipFile.Dispose(); - } - - // If the old file exists, delete it and replace - if (File.Exists(archiveFileName)) - { - FileTools.TryDeleteFile(archiveFileName); - } - File.Move(tempFile, archiveFileName); - - return true; - } - - #endregion } } diff --git a/SabreTools.Library/Tools/FileTools.cs b/SabreTools.Library/Tools/FileTools.cs index 0340356d..0b3d95cd 100644 --- a/SabreTools.Library/Tools/FileTools.cs +++ b/SabreTools.Library/Tools/FileTools.cs @@ -7,6 +7,7 @@ using System.Xml.Schema; using SabreTools.Library.Data; using SabreTools.Library.External; +using SabreTools.Library.FileTypes; using SabreTools.Library.Items; using SabreTools.Library.Skippers; diff --git a/SabreTools/SabreTools.Help.cs b/SabreTools/SabreTools.Help.cs index de7fd5ab..565b3ba7 100644 --- a/SabreTools/SabreTools.Help.cs +++ b/SabreTools/SabreTools.Help.cs @@ -468,11 +468,13 @@ namespace SabreTools FeatureType.Flag, null)); */ + /* sort.AddFeature("txz", new Feature( new List() { "-txz", "--txz" }, "Enable TorrentXZ output", FeatureType.Flag, null)); + */ sort.AddFeature("tzip", new Feature( new List() { "-tzip", "--tzip" }, "Enable TorrentZip output", @@ -604,11 +606,13 @@ namespace SabreTools FeatureType.Flag, null)); */ + /* sortDepot.AddFeature("txz", new Feature( new List() { "-txz", "--txz" }, "Enable TorrentXZ output", FeatureType.Flag, null)); + */ sortDepot.AddFeature("tzip", new Feature( new List() { "-tzip", "--tzip" }, "Enable TorrentZip output",