using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; using SabreTools.Library.Data; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; using SabreTools.Library.IO; using SabreTools.Library.Tools; using Compress; using Compress.gZip; using Compress.ZipFile.ZLib; namespace SabreTools.Library.FileTypes { /// /// Represents a TorrentGZip archive for reading and writing /// public class GZipArchive : BaseArchive { #region Constructors /// /// Create a new TorrentGZipArchive with no base file /// public GZipArchive() : base() { this.Type = FileType.GZipArchive; } /// /// Create a new TorrentGZipArchive from the given file /// /// Name of the file to use as an archive /// True for opening file as read, false for opening file as write /// True if hashes for this file should be calculated, false otherwise (default) public GZipArchive(string filename, bool getHashes = false) : base(filename, getHashes) { this.Type = FileType.GZipArchive; } #endregion #region Extraction /// /// Attempt to extract a file as an archive /// /// Output directory for archive extraction /// True if the extraction was a success, false otherwise public override bool CopyAll(string outDir) { bool encounteredErrors = true; try { // Create the temp directory Directory.CreateDirectory(outDir); // Decompress the _filename stream FileStream outstream = FileExtensions.TryCreate(Path.Combine(outDir, Path.GetFileNameWithoutExtension(this.Filename))); var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); gzstream.CopyTo(outstream); // Dispose of the streams outstream.Dispose(); ret = gz.ZipFileCloseReadStream(); gz.ZipFileClose(); encounteredErrors = false; } catch (EndOfStreamException ex) { // Catch this but don't count it as an error because SharpCompress is unsafe Globals.Logger.Verbose(ex); } catch (InvalidOperationException ex) { Globals.Logger.Warning(ex); encounteredErrors = true; } catch (Exception ex) { Globals.Logger.Error(ex); encounteredErrors = true; } return encounteredErrors; } /// /// Attempt to extract a file from an archive /// /// Name of the entry to be extracted /// Output directory for archive extraction /// Name of the extracted file, null on error public override string CopyToFile(string entryName, string outDir) { // Try to extract a stream using the given information (MemoryStream ms, string realEntry) = CopyToStream(entryName); // If the memory stream and the entry name are both non-null, we write to file if (ms != null && realEntry != null) { realEntry = Path.Combine(outDir, realEntry); // Create the output subfolder now Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); // Now open and write the file if possible FileStream fs = FileExtensions.TryCreate(realEntry); if (fs != null) { ms.Seek(0, SeekOrigin.Begin); byte[] zbuffer = new byte[_bufferSize]; int zlen; while ((zlen = ms.Read(zbuffer, 0, _bufferSize)) > 0) { fs.Write(zbuffer, 0, zlen); fs.Flush(); } ms?.Dispose(); fs?.Dispose(); } else { ms?.Dispose(); fs?.Dispose(); realEntry = null; } } return realEntry; } /// /// Attempt to extract a stream from an archive /// /// Name of the entry to be extracted /// Output representing the entry name that was found /// MemoryStream representing the entry, null on error public override (MemoryStream, string) CopyToStream(string entryName) { MemoryStream ms = new MemoryStream(); string realEntry; try { // Decompress the _filename stream realEntry = Path.GetFileNameWithoutExtension(this.Filename); var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); // Write the file out byte[] gbuffer = new byte[_bufferSize]; int glen; while ((glen = gzstream.Read(gbuffer, 0, _bufferSize)) > 0) { ms.Write(gbuffer, 0, glen); ms.Flush(); } // Dispose of the streams gzstream.Dispose(); } catch (Exception ex) { Globals.Logger.Error(ex); ms = null; realEntry = null; } return (ms, realEntry); } #endregion #region Information /// /// Generate a list of DatItem objects from the header values in an archive /// /// Hash representing the hashes that should be skipped /// True if entry dates should be included, false otherwise (default) /// List of DatItem objects representing the found data /// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually public override List GetChildren(Hash omitFromScan = Hash.DeepHashes, bool date = false) { if (_children == null || _children.Count == 0) { _children = new List(); string gamename = Path.GetFileNameWithoutExtension(this.Filename); BaseFile possibleTgz = GetTorrentGZFileInfo(); // If it was, then add it to the outputs and continue if (possibleTgz != null && possibleTgz.Filename != null) { _children.Add(possibleTgz); } else { try { // If secure hashes are disabled, do a quickscan if (omitFromScan == Hash.SecureHashes) { BaseFile tempRom = new BaseFile() { Filename = gamename, }; BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename)); br.BaseStream.Seek(-8, SeekOrigin.End); tempRom.CRC = br.ReadBytesBigEndian(4); tempRom.Size = br.ReadInt32BigEndian(); br.Dispose(); _children.Add(tempRom); } // Otherwise, use the stream directly else { var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); BaseFile gzipEntryRom = gzstream.GetInfo(); gzipEntryRom.RemoveHashes(omitFromScan); gzipEntryRom.Filename = gz.Filename(0); gzipEntryRom.Parent = gamename; gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null); _children.Add(gzipEntryRom); gzstream.Dispose(); } } catch (Exception ex) { Globals.Logger.Error(ex); return null; } } } return _children; } /// /// Generate a list of empty folders in an archive /// /// Input file to get data from /// List of empty folders in the archive public override List GetEmptyFolders() { // GZip files don't contain directories return new List(); } /// /// Check whether the input file is a standardized format /// public override bool IsTorrent() { // Check for the file existing first if (!File.Exists(this.Filename)) return false; string datum = Path.GetFileName(this.Filename).ToLowerInvariant(); long filesize = new FileInfo(this.Filename).Length; // If we have the romba depot files, just skip them gracefully if (datum == ".romba_size" || datum == ".romba_size.backup") { Globals.Logger.Verbose($"Romba depot file found, skipping: {this.Filename}"); return false; } // Check if the name is the right length if (!Regex.IsMatch(datum, @"^[0-9a-f]{" + Constants.SHA1Length + @"}\.gz")) { Globals.Logger.Warning($"Non SHA-1 filename found, skipping: '{Path.GetFullPath(this.Filename)}'"); return false; } // Check if the file is at least the minimum length if (filesize < 40 /* bytes */) { Globals.Logger.Warning($"Possibly corrupt file '{Path.GetFullPath(this.Filename)}' with size {Utilities.GetBytesReadable(filesize)}"); return false; } // Get the Romba-specific header data BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename)); byte[] header = br.ReadBytes(12); // Get preamble header for checking br.ReadBytes(16); // headermd5 br.ReadBytes(4); // headercrc br.ReadUInt64(); // headersz br.Dispose(); // If the header is not correct, return a blank rom bool correct = true; for (int i = 0; i < header.Length; i++) { // This is a temp fix to ignore the modification time and OS until romba can be fixed if (i == 4 || i == 5 || i == 6 || i == 7 || i == 9) continue; correct &= (header[i] == Constants.TorrentGZHeader[i]); } if (!correct) return false; return true; } /// /// Retrieve file information for a single torrent GZ file /// /// Populated DatItem object if success, empty one on error public BaseFile GetTorrentGZFileInfo() { // Check for the file existing first if (!File.Exists(this.Filename)) { return null; } string datum = Path.GetFileName(this.Filename).ToLowerInvariant(); long filesize = new FileInfo(this.Filename).Length; // If we have the romba depot files, just skip them gracefully if (datum == ".romba_size" || datum == ".romba_size.backup") { Globals.Logger.Verbose($"Romba depot file found, skipping: {this.Filename}"); return null; } // Check if the name is the right length if (!Regex.IsMatch(datum, @"^[0-9a-f]{" + Constants.SHA1Length + @"}\.gz")) { Globals.Logger.Warning($"Non SHA-1 filename found, skipping: '{Path.GetFullPath(this.Filename)}'"); return null; } // Check if the file is at least the minimum length if (filesize < 40 /* bytes */) { Globals.Logger.Warning($"Possibly corrupt file '{Path.GetFullPath(this.Filename)}' with size {Utilities.GetBytesReadable(filesize)}"); return null; } // Get the Romba-specific header data byte[] header; // Get preamble header for checking byte[] headermd5; // MD5 byte[] headercrc; // CRC ulong headersz; // Int64 size BinaryReader br = new BinaryReader(FileExtensions.TryOpenRead(this.Filename)); header = br.ReadBytes(12); headermd5 = br.ReadBytes(16); headercrc = br.ReadBytes(4); headersz = br.ReadUInt64(); br.Dispose(); // If the header is not correct, return a blank rom bool correct = true; for (int i = 0; i < header.Length; i++) { // This is a temp fix to ignore the modification time and OS until romba can be fixed if (i == 4 || i == 5 || i == 6 || i == 7 || i == 9) { continue; } correct &= (header[i] == Constants.TorrentGZHeader[i]); } if (!correct) { return null; } // Now convert the data and get the right position long extractedsize = (long)headersz; BaseFile baseFile = new BaseFile { Filename = Path.GetFileNameWithoutExtension(this.Filename).ToLowerInvariant(), Size = extractedsize, CRC = headercrc, MD5 = headermd5, SHA1 = Utilities.StringToByteArray(Path.GetFileNameWithoutExtension(this.Filename)), Parent = Path.GetFileNameWithoutExtension(this.Filename).ToLowerInvariant(), }; return baseFile; } #endregion #region Writing /// /// Write an input file to a torrent GZ file /// /// Input filename to be moved /// Output directory to build to /// DatItem representing the new information /// True if the date from the DAT should be used if available, false otherwise (default) /// Positive value for depth of the output depot, defaults to 4 /// True if the write was a success, false otherwise /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. public override bool Write(string inputFile, string outDir, Rom rom = null, bool date = false, int depth = 4) { // Check that the input file exists if (!File.Exists(inputFile)) { Globals.Logger.Warning($"File '{inputFile}' does not exist!"); return false; } inputFile = Path.GetFullPath(inputFile); // Get the file stream for the file and write out return Write(FileExtensions.TryOpenRead(inputFile), outDir, rom, date, depth); } /// /// Write an input stream to a torrent GZ file /// /// Input stream to be moved /// Output directory to build to /// DatItem representing the new information /// True if the date from the DAT should be used if available, false otherwise (default) /// Positive value for depth of the output depot, defaults to 4 /// True if the write was a success, false otherwise /// This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code. public override bool Write(Stream inputStream, string outDir, Rom rom = null, bool date = false, int depth = 4) { bool success = false; // If the stream is not readable, return if (!inputStream.CanRead) return success; // Make sure the output directory exists if (!Directory.Exists(outDir)) Directory.CreateDirectory(outDir); outDir = Path.GetFullPath(outDir); // Now get the Rom info for the file so we have hashes and size rom = new Rom(inputStream.GetInfo(keepReadOpen: true)); // Get the output file name string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, depth)); // Check to see if the folder needs to be created if (!Directory.Exists(Path.GetDirectoryName(outfile))) Directory.CreateDirectory(Path.GetDirectoryName(outfile)); // If the output file exists, don't try to write again if (!File.Exists(outfile)) { // Compress the input stream FileStream outputStream = FileExtensions.TryCreate(outfile); // Open the output file for writing BinaryWriter sw = new BinaryWriter(outputStream); // Write standard header and TGZ info byte[] data = Constants.TorrentGZHeader .Concat(Utilities.StringToByteArray(rom.MD5)) // MD5 .Concat(Utilities.StringToByteArray(rom.CRC)) // CRC .ToArray(); sw.Write(data); sw.Write((ulong)(rom.Size ?? 0)); // Long size (Unsigned, Mirrored) // Now create a deflatestream from the input file ZlibBaseStream ds = new ZlibBaseStream(outputStream, CompressionMode.Compress, CompressionLevel.BestCompression, ZlibStreamFlavor.DEFLATE, true); // Copy the input stream to the output byte[] ibuffer = new byte[_bufferSize]; int ilen; while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0) { ds.Write(ibuffer, 0, ilen); ds.Flush(); } ds.Dispose(); // Now write the standard footer sw.Write(Utilities.StringToByteArray(rom.CRC).Reverse().ToArray()); sw.Write((uint)(rom.Size ?? 0)); // Dispose of everything sw.Dispose(); outputStream.Dispose(); } return true; } /// /// Write a set of input files to a torrent GZ archive (assuming the same output archive name) /// /// Input files to be moved /// Output directory to build to /// DatItem representing the new information /// True if the date from the DAT should be used if available, false otherwise (default) /// True if files should be output in Romba depot folders, false otherwise /// True if the archive was written properly, false otherwise public override bool Write(List inputFiles, string outDir, List roms, bool date = false, bool romba = false) { throw new NotImplementedException(); } #endregion } }