diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index 9d8df223..360deb4b 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -47,54 +47,50 @@ namespace SabreTools List datItems = df.Files[key]; foreach (Rom rom in datItems) { - bool add = false; - if (onlyNeeded) - { - string query = @" -SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1 - JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1 -WHERE crcsha1.crc=""" + rom.CRC + @""" - OR md5sha1.md5=""" + rom.MD5 + @""" - OR md5sha1.sha1=""" + rom.SHA1 + "\""; - SqliteCommand slc = new SqliteCommand(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); + string query = "SELECT id FROM data WHERE size=" + rom.Size + + " AND (crc=\"" + rom.CRC + "\" OR crc=\"null\")" + + " AND (md5=\"" + rom.MD5 + "\" OR md5=\"null\")" + + " AND (sha1=\"" + rom.SHA1 + "\" OR sha1=\"null\")" + + " AND indepot=0"; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); - // If a row is returned, add the file and change the existence - if (sldr.HasRows) + // If a row is returned, add the file and change the existence + if (sldr.HasRows) + { + sldr.Read(); + long id = sldr.GetInt64(0); + + string squery = "UPDATE data SET indepot=1 WHERE id=" + id; + SqliteCommand sslc = new SqliteCommand(squery, dbc); + sslc.ExecuteNonQuery(); + sslc.Dispose(); + + // Add the rom to the files that need to be rebuilt + if (need.Files.ContainsKey(key)) { - add = true; + need.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + need.Files.Add(key, temp); } } - else + + // If it doesn't exist, and we're not adding only needed files + else if (!onlyNeeded) { - add = true; - } - - // If we're supposed to add the file to the depot, add it to the list - if (add) - { - _logger.User("Adding file \"" + rom.Name + "\" to the database"); - - // Insert new or updated information into the database - string query = "INSERT OR IGNORE INTO crc (crc) VALUES (\"" + rom.CRC + "\")"; - SqliteCommand slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); - - query = "INSERT OR IGNORE INTO md5 (md5) VALUES (\"" + rom.MD5 + "\")"; - slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); - - query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES (\"" + rom.SHA1 + "\")"; - slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); - - query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\")"; - slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); - - query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\")"; - slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); + string squery = "INSERT INTO data (size, crc, md5, sha1, indepot) VALUES (" + + rom.Size + "," + + "\"" + (rom.CRC == "" ? "null" : rom.CRC) + "\"," + + "\"" + (rom.MD5 == "" ? "null" : rom.MD5) + "\"," + + "\"" + (rom.SHA1 == "" ? "null" : rom.SHA1) + "\"," + + "1)"; + SqliteCommand sslc = new SqliteCommand(squery, dbc); + sslc.ExecuteNonQuery(); + sslc.Dispose(); // Add the rom to the files that need to be rebuilt if (need.Files.ContainsKey(key)) @@ -113,7 +109,7 @@ WHERE crcsha1.crc=""" + rom.CRC + @""" // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(0, 0, 0, 0); - SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, _logger); + SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, false, true, true, asl, false, _logger); ss.StartProcessing(); } @@ -152,12 +148,57 @@ WHERE crcsha1.crc=""" + rom.CRC + @""" Directory.CreateDirectory(outputFolder); } - // Get the depots that are online - List onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList(); + // Then get all hashes associated with this DAT + string query = "SELECT sha1 FROM dats JOIN data ON dats.id=data.id WHERE hash=\"" + key + "\""; + SqliteCommand slc = new SqliteCommand(query, dbc); + SqliteDataReader sldr = slc.ExecuteReader(); + if (sldr.HasRows) + { + while (sldr.Read()) + { + string sha1 = sldr.GetString(0); + string filename = Path.Combine(sha1.Substring(0, 2), sha1.Substring(2, 2), sha1.Substring(4, 2), sha1.Substring(6, 2), sha1 + ".gz"); - // Loop over the depots and rebuild as needed - SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, false, false, false, false, copy, copy, asl, false, _logger); - ss.StartProcessing(); + // Find the first depot that contains the folder + foreach (string depot in _depots.Keys) + { + // If the depot is online, check it + if (_depots[depot].Item2) + { + if (File.Exists(Path.Combine(depot, filename))) + { + if (copy) + { + if (!Directory.Exists(Path.Combine(outputFolder, Path.GetDirectoryName(filename)))) + { + Directory.CreateDirectory(Path.Combine(outputFolder, Path.GetDirectoryName(filename))); + } + + try + { + File.Copy(Path.Combine(depot, filename), Path.Combine(outputFolder, filename), true); + } + catch { } + } + else + { + ArchiveTools.ExtractArchive(Path.Combine(depot, filename), _tmpdir, asl, _logger); + } + continue; + } + } + } + } + } + + // Now that we have extracted everything, we rebuild to the output folder + if (!copy) + { + List temp = new List(); + temp.Add(_tmpdir); + SimpleSort ss = new SimpleSort(datFile, temp, outputFolder, "", false, false, false, false, true, false, false, asl, false, _logger); + ss.StartProcessing(); + } } dbc.Dispose(); @@ -264,7 +305,7 @@ WHERE crcsha1.crc=""" + rom.CRC + @""" // Now, search for each of them and return true or false for each foreach (string input in crc) { - string query = "SELECT * FROM crc WHERE crc=\"" + input + "\""; + string query = "SELECT * FROM data WHERE crc=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -281,7 +322,7 @@ WHERE crcsha1.crc=""" + rom.CRC + @""" } foreach (string input in md5) { - string query = "SELECT * FROM md5 WHERE md5=\"" + input + "\""; + string query = "SELECT * FROM data WHERE md5=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) @@ -298,7 +339,7 @@ WHERE crcsha1.crc=""" + rom.CRC + @""" } foreach (string input in sha1) { - string query = "SELECT * FROM sha1 WHERE sha1=\"" + input + "\""; + string query = "SELECT * FROM data WHERE sha1=\"" + input + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index 72b99e43..3e6dc7e9 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -285,6 +285,7 @@ namespace SabreTools.Helper helptext.Add(" -t=, --temp= Set the temporary directory to use"); helptext.Add(" -d, --delete Delete input files"); helptext.Add(" -qs, --quick Enable quick scanning of archives"); + helptext.Add(" -ad, --add-date Add original dates from DAT, if possible"); helptext.Add(" -v, --verify Enable verification of output directory"); helptext.Add(" -c, --convert Enable conversion of input files to TGZ"); helptext.Add(" Note: If a DAT is used, only files NOT included will rebuild"); diff --git a/SabreTools.Helper/Data/Constants.cs b/SabreTools.Helper/Data/Constants.cs index a12f2847..c6487d3d 100644 --- a/SabreTools.Helper/Data/Constants.cs +++ b/SabreTools.Helper/Data/Constants.cs @@ -181,6 +181,7 @@ namespace SabreTools.Helper public const uint EndOfCentralDirSignature = 0x06054b50; public const uint Zip64EndOfCentralDirSignature = 0x06064b50; public const uint Zip64EndOfCentralDirectoryLocator = 0x07064b50; + public const uint TorrentZipFileDateTime = 0x2198BC00; #endregion diff --git a/SabreTools.Helper/Objects/Archive/ZipFile.cs b/SabreTools.Helper/Objects/Archive/ZipFile.cs index a41d2014..7b8523f3 100644 --- a/SabreTools.Helper/Objects/Archive/ZipFile.cs +++ b/SabreTools.Helper/Objects/Archive/ZipFile.cs @@ -760,11 +760,12 @@ namespace SabreTools.Helper /// Size of the stream regardless of compression /// Compression method to compare against /// Status of the underlying stream - public ZipReturn OpenReadStream(int index, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod) + public ZipReturn OpenReadStream(int index, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod) { // Set all of the defaults streamSize = 0; compressionMethod = CompressionMethod.Stored; + lastMod = 0; _readIndex = index; stream = null; @@ -783,7 +784,7 @@ namespace SabreTools.Helper } // Now return the results of opening the local file - return _entries[index].OpenReadStream(raw, out stream, out streamSize, out compressionMethod); + return _entries[index].OpenReadStream(raw, out stream, out streamSize, out compressionMethod, out lastMod); } /// @@ -795,7 +796,7 @@ namespace SabreTools.Helper /// Size of the stream regardless of compression /// Compression method to compare against /// Status of the underlying stream - public ZipReturn OpenReadStreamQuick(ulong pos, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod) + public ZipReturn OpenReadStreamQuick(ulong pos, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod) { // Get the temporary entry based on the defined position ZipFileEntry tempEntry = new ZipFileEntry(_zipstream); @@ -812,12 +813,13 @@ namespace SabreTools.Helper stream = null; streamSize = 0; compressionMethod = CompressionMethod.Stored; + lastMod = 0; return zr; } _readIndex = 0; // Return the file stream if it worked - return tempEntry.OpenReadStream(raw, out stream, out streamSize, out compressionMethod); + return tempEntry.OpenReadStream(raw, out stream, out streamSize, out compressionMethod, out lastMod); } /// @@ -838,7 +840,8 @@ namespace SabreTools.Helper /// Compression method to compare against /// Output stream representing the correctly compressed stream /// Status of the underlying stream - public ZipReturn OpenWriteStream(bool raw, bool torrentZip, string filename, ulong uncompressedSize, CompressionMethod compressionMethod, out Stream stream) + public ZipReturn OpenWriteStream(bool raw, bool torrentZip, string filename, ulong uncompressedSize, + CompressionMethod compressionMethod, out Stream stream, uint lastMod = Constants.TorrentZipFileDateTime) { // Check to see if the stream is writable stream = null; @@ -848,7 +851,7 @@ namespace SabreTools.Helper } // Open the entry stream based on the current position - ZipFileEntry zfe = new ZipFileEntry(_zipstream, filename); + ZipFileEntry zfe = new ZipFileEntry(_zipstream, filename, lastMod: lastMod); ZipReturn zr = zfe.OpenWriteStream(raw, torrentZip, uncompressedSize, compressionMethod, out stream); _entries.Add(zfe); diff --git a/SabreTools.Helper/Objects/Archive/ZipFileEntry.cs b/SabreTools.Helper/Objects/Archive/ZipFileEntry.cs index 0ec7bfba..756ddf6a 100644 --- a/SabreTools.Helper/Objects/Archive/ZipFileEntry.cs +++ b/SabreTools.Helper/Objects/Archive/ZipFileEntry.cs @@ -23,8 +23,7 @@ namespace SabreTools.Helper private ArchiveVersion _versionMadeBy; private ArchiveVersion _versionNeeded; private GeneralPurposeBitFlag _generalPurposeBitFlag; - private ushort _lastModFileTime; - private ushort _lastModFileDate; + private uint _lastMod; private uint _crc; private ulong _compressedSize; private ulong _uncompressedSize; @@ -56,15 +55,10 @@ namespace SabreTools.Helper get { return _generalPurposeBitFlag; } private set { _generalPurposeBitFlag = value; } } - public ushort LastModFileTime + public uint LastMod { - get { return _lastModFileTime; } - set { _lastModFileTime = value; } - } - public ushort LastModFileDate - { - get { return _lastModFileDate; } - set { _lastModFileDate = value; } + get { return _lastMod; } + set { _lastMod = value; } } public byte[] CRC { @@ -135,14 +129,13 @@ namespace SabreTools.Helper /// /// Stream representing the entry /// Internal filename to use - public ZipFileEntry(Stream zipstream, string filename) + public ZipFileEntry(Stream zipstream, string filename, uint lastMod = Constants.TorrentZipFileDateTime) { _zip64 = false; _zipstream = zipstream; _generalPurposeBitFlag = GeneralPurposeBitFlag.DeflatingMaximumCompression; _compressionMethod = CompressionMethod.Deflated; - _lastModFileTime = 48128; - _lastModFileDate = 8600; + _lastMod = lastMod; FileName = filename; } @@ -181,8 +174,7 @@ namespace SabreTools.Helper } // Keep reading available information, skipping the unnecessary - _lastModFileTime = br.ReadUInt16(); - _lastModFileDate = br.ReadUInt16(); + _lastMod = br.ReadUInt32(); _crc = br.ReadUInt32(); _compressedSize = br.ReadUInt32(); _uncompressedSize = br.ReadUInt32(); @@ -356,8 +348,7 @@ namespace SabreTools.Helper bw.Write(versionNeededToExtract); bw.Write((ushort)_generalPurposeBitFlag); bw.Write((ushort)_compressionMethod); - bw.Write(_lastModFileTime); - bw.Write(_lastModFileDate); + bw.Write(_lastMod); bw.Write(_crc); bw.Write(compressedSize32); bw.Write(uncompressedSize32); @@ -413,11 +404,7 @@ namespace SabreTools.Helper { return ZipReturn.ZipLocalFileHeaderError; } - if (br.ReadUInt16() != _lastModFileTime) - { - return ZipReturn.ZipLocalFileHeaderError; - } - if (br.ReadUInt16() != _lastModFileDate) + if (br.ReadUInt32() != _lastMod) { return ZipReturn.ZipLocalFileHeaderError; } @@ -616,8 +603,7 @@ namespace SabreTools.Helper } _compressionMethod = (CompressionMethod)br.ReadUInt16(); - _lastModFileTime = br.ReadUInt16(); - _lastModFileDate = br.ReadUInt16(); + _lastMod = br.ReadUInt32(); _crc = br.ReadUInt32(); _compressedSize = br.ReadUInt32(); _uncompressedSize = br.ReadUInt32(); @@ -736,8 +722,7 @@ namespace SabreTools.Helper bw.Write(versionNeededToExtract); bw.Write((ushort)_generalPurposeBitFlag); bw.Write((ushort)_compressionMethod); - bw.Write(_lastModFileTime); - bw.Write(_lastModFileDate); + bw.Write(_lastMod); _crc32Location = (ulong)_zipstream.Position; @@ -780,10 +765,11 @@ namespace SabreTools.Helper /// Size of the stream regardless of compression /// Compression method to compare against /// Status of the underlying stream - public ZipReturn OpenReadStream(bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod) + public ZipReturn OpenReadStream(bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod) { streamSize = 0; compressionMethod = _compressionMethod; + lastMod = _lastMod; _readStream = null; _zipstream.Seek((long)_dataLocation, SeekOrigin.Begin); @@ -834,6 +820,7 @@ namespace SabreTools.Helper /// Uncompressed size of the stream /// Compression method to compare against /// Output stream representing the correctly compressed stream + /// True if the file should use the TorrentZip date (default), false otherwise /// Status of the underlying stream public ZipReturn OpenWriteStream(bool raw, bool torrentZip, ulong uncompressedSize, CompressionMethod compressionMethod, out Stream stream) { diff --git a/SabreTools.Helper/Objects/SimpleSort.cs b/SabreTools.Helper/Objects/SimpleSort.cs index 81bf4901..d1517b64 100644 --- a/SabreTools.Helper/Objects/SimpleSort.cs +++ b/SabreTools.Helper/Objects/SimpleSort.cs @@ -14,6 +14,7 @@ namespace SabreTools.Helper private string _outDir; private string _tempDir; private bool _quickScan; + private bool _date; private bool _toFolder; private bool _verify; private bool _delete; @@ -37,6 +38,7 @@ namespace SabreTools.Helper /// Output directory to use to build to /// Temporary directory for archive extraction /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise /// True if files should be output to folder, false otherwise /// True if output directory should be checked instead of rebuilt to, false otherwise /// True if input files should be deleted, false otherwise @@ -46,7 +48,7 @@ namespace SabreTools.Helper /// True if the updated DAT should be output, false otherwise /// Logger object for file and console output public SimpleSort(DatFile datdata, List inputs, string outDir, string tempDir, - bool quickScan, bool toFolder, bool verify, bool delete, bool tgz, bool romba, + bool quickScan, bool date, bool toFolder, bool verify, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, Logger logger) { _datdata = datdata; @@ -54,6 +56,7 @@ namespace SabreTools.Helper _outDir = (outDir == "" ? "Rebuild" : outDir); _tempDir = (String.IsNullOrEmpty(tempDir) ? Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) : tempDir); _quickScan = quickScan; + _date = date; _toFolder = toFolder; _verify = verify; _delete = delete; @@ -356,7 +359,7 @@ namespace SabreTools.Helper } else { - ArchiveTools.WriteToArchive(input, _outDir, found, _logger); + ArchiveTools.WriteToArchive(input, _outDir, found, _logger, date: _date); } } } @@ -373,7 +376,7 @@ namespace SabreTools.Helper Rom drom = FileTools.GetFileInfo(newinput, _logger); // If we have a blank RomData, it's an error - if (drom.Name == null) + if (String.IsNullOrEmpty(drom.Name)) { return false; } @@ -423,7 +426,7 @@ namespace SabreTools.Helper } else { - ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger); + ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger, date: _date); } } @@ -473,7 +476,7 @@ namespace SabreTools.Helper } else { - ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger); + ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger, date: _date); } } } diff --git a/SabreTools.Helper/README.1ST b/SabreTools.Helper/README.1ST index 70beeb51..0dd57187 100644 --- a/SabreTools.Helper/README.1ST +++ b/SabreTools.Helper/README.1ST @@ -709,6 +709,12 @@ Options: quicker than extracting all files to the temp folder. On the downside, it can only get the CRC and size from most archive formats, leading to possible issues. + -ad, --add-date Write dates for each file parsed, if available + If this flag is set, the the date in the DAT will be used for the output file + instead of the standard date and time for TorrentZip. This will technically + invalidate the output files as proper TorrentZip files because the date will not + match the standard. + -v, --verify Enable verification of output directory This overrides the default rebuilding and only requires the DAT and the output folder. Here, the DAT is used to verify the output directory directly and then output a diff --git a/SabreTools.Helper/Tools/ArchiveTools.cs b/SabreTools.Helper/Tools/ArchiveTools.cs index 164eef3d..f23035af 100644 --- a/SabreTools.Helper/Tools/ArchiveTools.cs +++ b/SabreTools.Helper/Tools/ArchiveTools.cs @@ -153,8 +153,9 @@ namespace SabreTools.Helper Stream readStream; ulong streamsize = 0; CompressionMethod cm = CompressionMethod.Stored; + uint lastMod = 0; - zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm); + zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm, out lastMod); FileStream writeStream = File.OpenWrite(Path.Combine(tempDir, zf.Entries[i].FileName)); @@ -290,8 +291,9 @@ namespace SabreTools.Helper Stream readStream; ulong streamsize = 0; CompressionMethod cm = CompressionMethod.Stored; + uint lastMod = 0; - zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm); + zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm, out lastMod); byte[] ibuffer = new byte[_bufferSize]; int ilen; @@ -745,8 +747,9 @@ namespace SabreTools.Helper /// Output directory to build to /// RomData representing the new information /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) /// True if the archive was written properly, false otherwise - public static bool WriteToArchive(string inputFile, string outDir, Rom rom, Logger logger) + public static bool WriteToArchive(string inputFile, string outDir, Rom rom, Logger logger, bool date = false) { // Wrap the individual inputs into lists List inputFiles = new List(); @@ -754,7 +757,7 @@ namespace SabreTools.Helper List roms = new List(); roms.Add(rom); - return WriteToArchive(inputFiles, outDir, roms, logger); + return WriteToArchive(inputFiles, outDir, roms, logger, date: date); } /// @@ -764,8 +767,9 @@ namespace SabreTools.Helper /// Output directory to build to /// List of Rom representing the new information /// Logger object for file and console output + /// True if the date from the DAT should be used if available, false otherwise (default) /// True if the archive was written properly, false otherwise - public static bool WriteToArchive(List inputFiles, string outDir, List roms, Logger logger) + public static bool WriteToArchive(List inputFiles, string outDir, List roms, Logger logger, bool date = false) { bool success = false; string tempFile = Path.GetTempFileName(); @@ -827,7 +831,26 @@ namespace SabreTools.Helper // Open the input file for reading Stream freadStream = File.Open(inputFiles[index], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length); - zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt)) + { + // https://referencesource.microsoft.com/#WindowsBase/Base/MS/Internal/IO/Zip/ZipIOBlockManager.cs,760eb7714f02c41e + uint msDosDateTime = 0; + msDosDateTime |= (((uint)dt.Second) / 2) & 0x1F; + msDosDateTime |= (((uint)dt.Minute) & 0x3F) << 5; + msDosDateTime |= (((uint)dt.Hour) & 0x1F) << 11; + msDosDateTime |= (((uint)dt.Day) & 0x1F) << 16; + msDosDateTime |= (((uint)dt.Month) & 0xF) << 21; + msDosDateTime |= (((uint)(dt.Year - 1980)) & 0x7F) << 25; + + zipFile.OpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize, + CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream); + } // Copy the input stream to the output byte[] ibuffer = new byte[_bufferSize]; @@ -893,7 +916,26 @@ namespace SabreTools.Helper // Open the input file for reading Stream freadStream = File.Open(inputFiles[-index - 1], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length); - zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream); + + DateTime dt = DateTime.Now; + if (date && !String.IsNullOrEmpty(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt)) + { + // https://referencesource.microsoft.com/#WindowsBase/Base/MS/Internal/IO/Zip/ZipIOBlockManager.cs,760eb7714f02c41e + uint msDosDateTime = 0; + msDosDateTime |= (((uint)dt.Second) / 2) & 0x1F; + msDosDateTime |= (((uint)dt.Minute) & 0x3F) << 5; + msDosDateTime |= (((uint)dt.Hour) & 0x1F) << 11; + msDosDateTime |= (((uint)dt.Day) & 0x1F) << 16; + msDosDateTime |= (((uint)dt.Month) & 0xF) << 21; + msDosDateTime |= (((uint)(dt.Year - 1980)) & 0x7F) << 25; + + zipFile.OpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, + CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime); + } + else + { + zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream); + } // Copy the input stream to the output byte[] ibuffer = new byte[_bufferSize]; @@ -912,10 +954,12 @@ namespace SabreTools.Helper { // Instantiate the streams CompressionMethod icompressionMethod = CompressionMethod.Stored; + uint lastMod = 0; ulong istreamSize = 0; Stream zreadStream; - oldZipFile.OpenReadStream(index, false, out zreadStream, out istreamSize, out icompressionMethod); - zipFile.OpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, CompressionMethod.Deflated, out writeStream); + oldZipFile.OpenReadStream(index, false, out zreadStream, out istreamSize, out icompressionMethod, out lastMod); + zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index), + istreamSize, CompressionMethod.Deflated, out writeStream, lastMod: lastMod); // Copy the input stream to the output byte[] ibuffer = new byte[_bufferSize]; diff --git a/SabreTools.Helper/Tools/FileTools.cs b/SabreTools.Helper/Tools/FileTools.cs index 1857a547..810af435 100644 --- a/SabreTools.Helper/Tools/FileTools.cs +++ b/SabreTools.Helper/Tools/FileTools.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Xml; using System.Xml.Schema; diff --git a/SabreTools.Helper/Tools/Style.cs b/SabreTools.Helper/Tools/Style.cs index 044c9990..e75becff 100644 --- a/SabreTools.Helper/Tools/Style.cs +++ b/SabreTools.Helper/Tools/Style.cs @@ -550,6 +550,30 @@ namespace SabreTools.Helper return hexOutput; } + /// + /// Adapted from 7-zip Source Code: CPP/Windows/TimeUtils.cpp:FileTimeToDosTime + /// + public static uint ConvertDateTimeToMsDosTimeFormat(DateTime dateTime) + { + uint year = (uint)((dateTime.Year - 1980) % 128); + uint mon = (uint)dateTime.Month; + uint day = (uint)dateTime.Day; + uint hour = (uint)dateTime.Hour; + uint min = (uint)dateTime.Minute; + uint sec = (uint)dateTime.Second; + + return (year << 25) | (mon << 21) | (day << 16) | (hour << 11) | (min << 5) | (sec >> 1); + } + + /// + /// Adapted from 7-zip Source Code: CPP/Windows/TimeUtils.cpp:DosTimeToFileTime + /// + public static DateTime ConvertMsDosTimeFormatToDateTime(uint msDosDateTime) + { + return new DateTime((int)(1980 + (msDosDateTime >> 25)), (int)((msDosDateTime >> 21) & 0xF), (int)((msDosDateTime >> 16) & 0x1F), + (int)((msDosDateTime >> 11) & 0x1F), (int)((msDosDateTime >> 5) & 0x3F), (int)((msDosDateTime & 0x1F) * 2)); + } + /// /// Determines a text file's encoding by analyzing its byte order mark (BOM). /// Defaults to ASCII when detection of the text file's endianness fails. diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index ad92a86f..f5b670d3 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -18,6 +18,7 @@ namespace SabreTools /// Output directory (empty for default directory) /// Temporary directory for archive extraction /// True if input files should be deleted, false otherwise + /// True to output files in TorrentGZ format, false for TorrentZip /// True if files should be output in Romba depot folders, false otherwise /// Integer representing the archive handling level for 7z /// Integer representing the archive handling level for GZip @@ -25,19 +26,25 @@ namespace SabreTools /// Integer representing the archive handling level for Zip /// Logger object for file and console output public static bool InitConvertFolder(List datfiles, List inputs, string outDir, string tempDir, bool delete, - bool romba, int sevenzip, int gz, int rar, int zip, Logger logger) + bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, Logger logger) { + // Get the archive scanning level + ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); + + DateTime start = DateTime.Now; + logger.User("Populating internal DAT..."); + // Add all of the input DATs into one huge internal DAT DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { datdata.Parse(datfile, 99, 99, logger, keep: true, softlist: true); } - - // Get the archive scanning level - ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); + logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); // Get all individual files from the inputs + start = DateTime.Now; + logger.User("Organizing input files..."); List newinputs = new List(); foreach (string input in inputs) { @@ -53,9 +60,10 @@ namespace SabreTools } } } + logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, false, - delete, false, romba, asl, false, logger); + SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, + false, false, delete, tgz, romba, asl, false, logger); return ss.Convert(); } @@ -285,32 +293,37 @@ namespace SabreTools /// Output directory to use to build to /// Temporary directory for archive extraction /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise /// Integer representing the archive handling level for 7z /// True if files should be output to folder, false otherwise /// True if output directory should be checked instead of rebuilt to, false otherwise /// True if input files should be deleted, false otherwise - /// True is for TorrentZip, False is for TorrentGZ, Null is for standard zip + /// True to output files in TorrentGZ format, false for TorrentZip /// True if files should be output in Romba depot folders, false otherwise /// Integer representing the archive handling level for GZip /// Integer representing the archive handling level for RAR /// Integer representing the archive handling level for Zip /// True if the updated DAT should be output, false otherwise /// Logger object for file and console output - private static void InitSortVerify(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, + private static void InitSortVerify(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, bool date, bool toFolder, bool verify, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, Logger logger) { // Get the archive scanning level ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); + DateTime start = DateTime.Now; + logger.User("Populating internal DAT..."); + // Add all of the input DATs into one huge internal DAT DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { datdata.Parse(datfile, 99, 99, logger, keep: true, softlist: true); } + logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, toFolder, verify, - delete, tgz, romba, asl, updateDat, logger); + SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, + toFolder, verify, delete, tgz, romba, asl, updateDat, logger); ss.StartProcessing(); } diff --git a/SimpleSort/SimpleSortApp.cs b/SimpleSort/SimpleSortApp.cs index 786501e4..979cd08b 100644 --- a/SimpleSort/SimpleSortApp.cs +++ b/SimpleSort/SimpleSortApp.cs @@ -43,6 +43,7 @@ namespace SabreTools // Set all default values bool help = false, convert = false, + date = false, delete = false, quickScan = false, romba = false, @@ -70,6 +71,10 @@ namespace SabreTools case "--help": help = true; break; + case "-ad": + case "--add-date": + date = true; + break; case "-c": case "--convert": convert = true; @@ -201,7 +206,7 @@ namespace SabreTools { if (datfiles.Count > 0) { - InitSortVerify(datfiles, inputs, outDir, tempDir, quickScan, toFolder, + InitSortVerify(datfiles, inputs, outDir, tempDir, quickScan, date, toFolder, verify, delete, tgz, romba, sevenzip, gz, rar, zip, updateDat, logger); } else @@ -275,8 +280,8 @@ namespace SabreTools } logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, false, - delete, tgz, romba, asl, false, logger); + SimpleSort ss = new SimpleSort(datdata, newinputs, outDir, tempDir, false, false, + false, false, delete, tgz, romba, asl, false, logger); return ss.Convert(); } @@ -288,6 +293,7 @@ namespace SabreTools /// Output directory to use to build to /// Temporary directory for archive extraction /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise /// Integer representing the archive handling level for 7z /// True if files should be output to folder, false otherwise /// True if output directory should be checked instead of rebuilt to, false otherwise @@ -299,7 +305,7 @@ namespace SabreTools /// Integer representing the archive handling level for Zip /// True if the updated DAT should be output, false otherwise /// Logger object for file and console output - private static void InitSortVerify(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, + private static void InitSortVerify(List datfiles, List inputs, string outDir, string tempDir, bool quickScan, bool date, bool toFolder, bool verify, bool delete, bool tgz, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, Logger logger) { // Get the archive scanning level @@ -316,8 +322,8 @@ namespace SabreTools } logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, toFolder, verify, - delete, tgz, romba, asl, updateDat, logger); + SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, + toFolder, verify, delete, tgz, romba, asl, updateDat, logger); ss.StartProcessing(); } }