claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ALL] Allow for original Dates to be written again in zipfiles

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-14 16:58:15 -07:00
parent cbbab3518c
commit 1a998d575c
12 changed files with 245 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
SabreTools.Helper/Data/Build.cs
View File

@@ -285,6 +285,7 @@ namespace SabreTools.Helper
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -d, --delete Delete input files");
helptext.Add(" -qs, --quick Enable quick scanning of archives");
helptext.Add(" -ad, --add-date Add original dates from DAT, if possible");
helptext.Add(" -v, --verify Enable verification of output directory");
helptext.Add(" -c, --convert Enable conversion of input files to TGZ");
helptext.Add(" Note: If a DAT is used, only files NOT included will rebuild");

1
SabreTools.Helper/Data/Constants.cs
View File

@@ -181,6 +181,7 @@ namespace SabreTools.Helper
public const uint EndOfCentralDirSignature = 0x06054b50;
public const uint Zip64EndOfCentralDirSignature = 0x06064b50;
public const uint Zip64EndOfCentralDirectoryLocator = 0x07064b50;
public const uint TorrentZipFileDateTime = 0x2198BC00;
#endregion

15
SabreTools.Helper/Objects/Archive/ZipFile.cs
View File

@@ -760,11 +760,12 @@ namespace SabreTools.Helper
/// <param name="streamSize">Size of the stream regardless of compression</param>
/// <param name="compressionMethod">Compression method to compare against</param>
/// <returns>Status of the underlying stream</returns>
public ZipReturn OpenReadStream(int index, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod)
public ZipReturn OpenReadStream(int index, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod)
{
// Set all of the defaults
streamSize = 0;
compressionMethod = CompressionMethod.Stored;
lastMod = 0;
_readIndex = index;
stream = null;
@@ -783,7 +784,7 @@ namespace SabreTools.Helper
}
// Now return the results of opening the local file
return _entries[index].OpenReadStream(raw, out stream, out streamSize, out compressionMethod);
return _entries[index].OpenReadStream(raw, out stream, out streamSize, out compressionMethod, out lastMod);
}
/// <summary>
@@ -795,7 +796,7 @@ namespace SabreTools.Helper
/// <param name="streamSize">Size of the stream regardless of compression</param>
/// <param name="compressionMethod">Compression method to compare against</param>
/// <returns>Status of the underlying stream</returns>
public ZipReturn OpenReadStreamQuick(ulong pos, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod)
public ZipReturn OpenReadStreamQuick(ulong pos, bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod)
{
// Get the temporary entry based on the defined position
ZipFileEntry tempEntry = new ZipFileEntry(_zipstream);
@@ -812,12 +813,13 @@ namespace SabreTools.Helper
stream = null;
streamSize = 0;
compressionMethod = CompressionMethod.Stored;
lastMod = 0;
return zr;
}
_readIndex = 0;
// Return the file stream if it worked
return tempEntry.OpenReadStream(raw, out stream, out streamSize, out compressionMethod);
return tempEntry.OpenReadStream(raw, out stream, out streamSize, out compressionMethod, out lastMod);
}
/// <summary>
@@ -838,7 +840,8 @@ namespace SabreTools.Helper
/// <param name="compressionMethod">Compression method to compare against</param>
/// <param name="stream">Output stream representing the correctly compressed stream</param>
/// <returns>Status of the underlying stream</returns>
public ZipReturn OpenWriteStream(bool raw, bool torrentZip, string filename, ulong uncompressedSize, CompressionMethod compressionMethod, out Stream stream)
public ZipReturn OpenWriteStream(bool raw, bool torrentZip, string filename, ulong uncompressedSize,
CompressionMethod compressionMethod, out Stream stream, uint lastMod = Constants.TorrentZipFileDateTime)
{
// Check to see if the stream is writable
stream = null;
@@ -848,7 +851,7 @@ namespace SabreTools.Helper
}
// Open the entry stream based on the current position
ZipFileEntry zfe = new ZipFileEntry(_zipstream, filename);
ZipFileEntry zfe = new ZipFileEntry(_zipstream, filename, lastMod: lastMod);
ZipReturn zr = zfe.OpenWriteStream(raw, torrentZip, uncompressedSize, compressionMethod, out stream);
_entries.Add(zfe);

41
SabreTools.Helper/Objects/Archive/ZipFileEntry.cs
View File

@@ -23,8 +23,7 @@ namespace SabreTools.Helper
private ArchiveVersion _versionMadeBy;
private ArchiveVersion _versionNeeded;
private GeneralPurposeBitFlag _generalPurposeBitFlag;
private ushort _lastModFileTime;
private ushort _lastModFileDate;
private uint _lastMod;
private uint _crc;
private ulong _compressedSize;
private ulong _uncompressedSize;
@@ -56,15 +55,10 @@ namespace SabreTools.Helper
get { return _generalPurposeBitFlag; }
private set { _generalPurposeBitFlag = value; }
}
public ushort LastModFileTime
public uint LastMod
{
get { return _lastModFileTime; }
set { _lastModFileTime = value; }
}
public ushort LastModFileDate
{
get { return _lastModFileDate; }
set { _lastModFileDate = value; }
get { return _lastMod; }
set { _lastMod = value; }
}
public byte[] CRC
{
@@ -135,14 +129,13 @@ namespace SabreTools.Helper
/// </summary>
/// <param name="zipstream">Stream representing the entry</param>
/// <param name="filename">Internal filename to use</param>
public ZipFileEntry(Stream zipstream, string filename)
public ZipFileEntry(Stream zipstream, string filename, uint lastMod = Constants.TorrentZipFileDateTime)
{
_zip64 = false;
_zipstream = zipstream;
_generalPurposeBitFlag = GeneralPurposeBitFlag.DeflatingMaximumCompression;
_compressionMethod = CompressionMethod.Deflated;
_lastModFileTime = 48128;
_lastModFileDate = 8600;
_lastMod = lastMod;
FileName = filename;
}
@@ -181,8 +174,7 @@ namespace SabreTools.Helper
}
// Keep reading available information, skipping the unnecessary
_lastModFileTime = br.ReadUInt16();
_lastModFileDate = br.ReadUInt16();
_lastMod = br.ReadUInt32();
_crc = br.ReadUInt32();
_compressedSize = br.ReadUInt32();
_uncompressedSize = br.ReadUInt32();
@@ -356,8 +348,7 @@ namespace SabreTools.Helper
bw.Write(versionNeededToExtract);
bw.Write((ushort)_generalPurposeBitFlag);
bw.Write((ushort)_compressionMethod);
bw.Write(_lastModFileTime);
bw.Write(_lastModFileDate);
bw.Write(_lastMod);
bw.Write(_crc);
bw.Write(compressedSize32);
bw.Write(uncompressedSize32);
@@ -413,11 +404,7 @@ namespace SabreTools.Helper
{
return ZipReturn.ZipLocalFileHeaderError;
}
if (br.ReadUInt16() != _lastModFileTime)
{
return ZipReturn.ZipLocalFileHeaderError;
}
if (br.ReadUInt16() != _lastModFileDate)
if (br.ReadUInt32() != _lastMod)
{
return ZipReturn.ZipLocalFileHeaderError;
}
@@ -616,8 +603,7 @@ namespace SabreTools.Helper
}
_compressionMethod = (CompressionMethod)br.ReadUInt16();
_lastModFileTime = br.ReadUInt16();
_lastModFileDate = br.ReadUInt16();
_lastMod = br.ReadUInt32();
_crc = br.ReadUInt32();
_compressedSize = br.ReadUInt32();
_uncompressedSize = br.ReadUInt32();
@@ -736,8 +722,7 @@ namespace SabreTools.Helper
bw.Write(versionNeededToExtract);
bw.Write((ushort)_generalPurposeBitFlag);
bw.Write((ushort)_compressionMethod);
bw.Write(_lastModFileTime);
bw.Write(_lastModFileDate);
bw.Write(_lastMod);
_crc32Location = (ulong)_zipstream.Position;
@@ -780,10 +765,11 @@ namespace SabreTools.Helper
/// <param name="streamSize">Size of the stream regardless of compression</param>
/// <param name="compressionMethod">Compression method to compare against</param>
/// <returns>Status of the underlying stream</returns>
public ZipReturn OpenReadStream(bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod)
public ZipReturn OpenReadStream(bool raw, out Stream stream, out ulong streamSize, out CompressionMethod compressionMethod, out uint lastMod)
{
streamSize = 0;
compressionMethod = _compressionMethod;
lastMod = _lastMod;
_readStream = null;
_zipstream.Seek((long)_dataLocation, SeekOrigin.Begin);
@@ -834,6 +820,7 @@ namespace SabreTools.Helper
/// <param name="uncompressedSize">Uncompressed size of the stream</param>
/// <param name="compressionMethod">Compression method to compare against</param>
/// <param name="stream">Output stream representing the correctly compressed stream</param>
/// <param name="tzip">True if the file should use the TorrentZip date (default), false otherwise</param>
/// <returns>Status of the underlying stream</returns>
public ZipReturn OpenWriteStream(bool raw, bool torrentZip, ulong uncompressedSize, CompressionMethod compressionMethod, out Stream stream)
{

13
SabreTools.Helper/Objects/SimpleSort.cs
View File

@@ -14,6 +14,7 @@ namespace SabreTools.Helper
private string _outDir;
private string _tempDir;
private bool _quickScan;
private bool _date;
private bool _toFolder;
private bool _verify;
private bool _delete;
@@ -37,6 +38,7 @@ namespace SabreTools.Helper
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="toFolder">True if files should be output to folder, false otherwise</param>
/// <param name="verify">True if output directory should be checked instead of rebuilt to, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
@@ -46,7 +48,7 @@ namespace SabreTools.Helper
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
public SimpleSort(DatFile datdata, List<string> inputs, string outDir, string tempDir,
bool quickScan, bool toFolder, bool verify, bool delete, bool tgz, bool romba,
bool quickScan, bool date, bool toFolder, bool verify, bool delete, bool tgz, bool romba,
ArchiveScanLevel archiveScanLevel, bool updateDat, Logger logger)
{
_datdata = datdata;
@@ -54,6 +56,7 @@ namespace SabreTools.Helper
_outDir = (outDir == "" ? "Rebuild" : outDir);
_tempDir = (String.IsNullOrEmpty(tempDir) ? Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) : tempDir);
_quickScan = quickScan;
_date = date;
_toFolder = toFolder;
_verify = verify;
_delete = delete;
@@ -356,7 +359,7 @@ namespace SabreTools.Helper
}
else
{
ArchiveTools.WriteToArchive(input, _outDir, found, _logger);
ArchiveTools.WriteToArchive(input, _outDir, found, _logger, date: _date);
}
}
}
@@ -373,7 +376,7 @@ namespace SabreTools.Helper
Rom drom = FileTools.GetFileInfo(newinput, _logger);
// If we have a blank RomData, it's an error
if (drom.Name == null)
if (String.IsNullOrEmpty(drom.Name))
{
return false;
}
@@ -423,7 +426,7 @@ namespace SabreTools.Helper
}
else
{
ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger);
ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger, date: _date);
}
}
@@ -473,7 +476,7 @@ namespace SabreTools.Helper
}
else
{
ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger);
ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger, date: _date);
}
}
}

6
SabreTools.Helper/README.1ST
View File

@@ -709,6 +709,12 @@ Options:
quicker than extracting all files to the temp folder. On the downside, it can only
get the CRC and size from most archive formats, leading to possible issues.
-ad, --add-date Write dates for each file parsed, if available
If this flag is set, the the date in the DAT will be used for the output file
instead of the standard date and time for TorrentZip. This will technically
invalidate the output files as proper TorrentZip files because the date will not
match the standard.
-v, --verify Enable verification of output directory
This overrides the default rebuilding and only requires the DAT and the output folder.
Here, the DAT is used to verify the output directory directly and then output a

62
SabreTools.Helper/Tools/ArchiveTools.cs
View File

@@ -153,8 +153,9 @@ namespace SabreTools.Helper
Stream readStream;
ulong streamsize = 0;
CompressionMethod cm = CompressionMethod.Stored;
uint lastMod = 0;
zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm);
zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm, out lastMod);
FileStream writeStream = File.OpenWrite(Path.Combine(tempDir, zf.Entries[i].FileName));
@@ -290,8 +291,9 @@ namespace SabreTools.Helper
Stream readStream;
ulong streamsize = 0;
CompressionMethod cm = CompressionMethod.Stored;
uint lastMod = 0;
zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm);
zr = zf.OpenReadStream(i, false, out readStream, out streamsize, out cm, out lastMod);
byte[] ibuffer = new byte[_bufferSize];
int ilen;
@@ -745,8 +747,9 @@ namespace SabreTools.Helper
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">RomData representing the new information</param>
/// <param name="logger">Logger object for file and console output</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise (default)</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public static bool WriteToArchive(string inputFile, string outDir, Rom rom, Logger logger)
public static bool WriteToArchive(string inputFile, string outDir, Rom rom, Logger logger, bool date = false)
{
// Wrap the individual inputs into lists
List<string> inputFiles = new List<string>();
@@ -754,7 +757,7 @@ namespace SabreTools.Helper
List<Rom> roms = new List<Rom>();
roms.Add(rom);
return WriteToArchive(inputFiles, outDir, roms, logger);
return WriteToArchive(inputFiles, outDir, roms, logger, date: date);
}
/// <summary>
@@ -764,8 +767,9 @@ namespace SabreTools.Helper
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">List of Rom representing the new information</param>
/// <param name="logger">Logger object for file and console output</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise (default)</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public static bool WriteToArchive(List<string> inputFiles, string outDir, List<Rom> roms, Logger logger)
public static bool WriteToArchive(List<string> inputFiles, string outDir, List<Rom> roms, Logger logger, bool date = false)
{
bool success = false;
string tempFile = Path.GetTempFileName();
@@ -827,7 +831,26 @@ namespace SabreTools.Helper
// Open the input file for reading
Stream freadStream = File.Open(inputFiles[index], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length);
zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream);
DateTime dt = DateTime.Now;
if (date && !String.IsNullOrEmpty(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt))
{
// https://referencesource.microsoft.com/#WindowsBase/Base/MS/Internal/IO/Zip/ZipIOBlockManager.cs,760eb7714f02c41e
uint msDosDateTime = 0;
msDosDateTime |= (((uint)dt.Second) / 2) & 0x1F;
msDosDateTime |= (((uint)dt.Minute) & 0x3F) << 5;
msDosDateTime |= (((uint)dt.Hour) & 0x1F) << 11;
msDosDateTime |= (((uint)dt.Day) & 0x1F) << 16;
msDosDateTime |= (((uint)dt.Month) & 0xF) << 21;
msDosDateTime |= (((uint)(dt.Year - 1980)) & 0x7F) << 25;
zipFile.OpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize,
CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime);
}
else
{
zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
@@ -893,7 +916,26 @@ namespace SabreTools.Helper
// Open the input file for reading
Stream freadStream = File.Open(inputFiles[-index - 1], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length);
zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream);
DateTime dt = DateTime.Now;
if (date && !String.IsNullOrEmpty(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt))
{
// https://referencesource.microsoft.com/#WindowsBase/Base/MS/Internal/IO/Zip/ZipIOBlockManager.cs,760eb7714f02c41e
uint msDosDateTime = 0;
msDosDateTime |= (((uint)dt.Second) / 2) & 0x1F;
msDosDateTime |= (((uint)dt.Minute) & 0x3F) << 5;
msDosDateTime |= (((uint)dt.Hour) & 0x1F) << 11;
msDosDateTime |= (((uint)dt.Day) & 0x1F) << 16;
msDosDateTime |= (((uint)dt.Month) & 0xF) << 21;
msDosDateTime |= (((uint)(dt.Year - 1980)) & 0x7F) << 25;
zipFile.OpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize,
CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime);
}
else
{
zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, CompressionMethod.Deflated, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
@@ -912,10 +954,12 @@ namespace SabreTools.Helper
{
// Instantiate the streams
CompressionMethod icompressionMethod = CompressionMethod.Stored;
uint lastMod = 0;
ulong istreamSize = 0;
Stream zreadStream;
oldZipFile.OpenReadStream(index, false, out zreadStream, out istreamSize, out icompressionMethod);
zipFile.OpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, CompressionMethod.Deflated, out writeStream);
oldZipFile.OpenReadStream(index, false, out zreadStream, out istreamSize, out icompressionMethod, out lastMod);
zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index),
istreamSize, CompressionMethod.Deflated, out writeStream, lastMod: lastMod);
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];

1
SabreTools.Helper/Tools/FileTools.cs
View File

@@ -4,6 +4,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Xml;
using System.Xml.Schema;

24
SabreTools.Helper/Tools/Style.cs
View File

@@ -550,6 +550,30 @@ namespace SabreTools.Helper
return hexOutput;
}
/// <summary>
/// Adapted from 7-zip Source Code: CPP/Windows/TimeUtils.cpp:FileTimeToDosTime
/// </summary>
public static uint ConvertDateTimeToMsDosTimeFormat(DateTime dateTime)
{
uint year = (uint)((dateTime.Year - 1980) % 128);
uint mon = (uint)dateTime.Month;
uint day = (uint)dateTime.Day;
uint hour = (uint)dateTime.Hour;
uint min = (uint)dateTime.Minute;
uint sec = (uint)dateTime.Second;
return (year << 25) | (mon << 21) | (day << 16) | (hour << 11) | (min << 5) | (sec >> 1);
}
/// <summary>
/// Adapted from 7-zip Source Code: CPP/Windows/TimeUtils.cpp:DosTimeToFileTime
/// </summary>
public static DateTime ConvertMsDosTimeFormatToDateTime(uint msDosDateTime)
{
return new DateTime((int)(1980 + (msDosDateTime >> 25)), (int)((msDosDateTime >> 21) & 0xF), (int)((msDosDateTime >> 16) & 0x1F),
(int)((msDosDateTime >> 11) & 0x1F), (int)((msDosDateTime >> 5) & 0x3F), (int)((msDosDateTime & 0x1F) * 2));
}
/// <summary>
/// Determines a text file's encoding by analyzing its byte order mark (BOM).
/// Defaults to ASCII when detection of the text file's endianness fails.