Internal Fixes, etc. (#20)

* Start removing mixed usages

* Check for directories before opening

* Fix writing

* Kinda fix rebuild

* One more try

* Better internal handling

* Slighty fix a couple more things

* Update RVWorld Compress code to db7d750bba

* Fix build

Co-authored-by: Matt Nadareski <mnadareski@mparticle.com>
This commit is contained in:
Matt Nadareski
2020-04-03 13:19:21 -07:00
committed by GitHub
parent 1de4bc7b18
commit 916d2a3b51
79 changed files with 3377 additions and 2337 deletions

View File

@@ -4064,7 +4064,7 @@ namespace SabreTools.Library.DatFiles
foreach (BaseFile entry in entries) foreach (BaseFile entry in entries)
{ {
DatItem datItem = Utilities.GetDatItem(entry); DatItem datItem = Utilities.GetDatItem(entry);
usedInternally &= RebuildIndividualFile(datItem, file, outDir, date, inverse, outputFormat, usedInternally |= RebuildIndividualFile(datItem, file, outDir, date, inverse, outputFormat,
romba, updateDat, !isTorrentGzip /* isZip */, headerToCheckAgainst); romba, updateDat, !isTorrentGzip /* isZip */, headerToCheckAgainst);
} }
} }
@@ -4094,8 +4094,8 @@ namespace SabreTools.Library.DatFiles
private bool RebuildIndividualFile(DatItem datItem, string file, string outDir, bool date, private bool RebuildIndividualFile(DatItem datItem, string file, string outDir, bool date,
bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool? isZip, string headerToCheckAgainst) bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool? isZip, string headerToCheckAgainst)
{ {
// Set the output value // Set the initial output value
bool rebuilt = true; bool rebuilt = false;
// If the DatItem is a Disk, force rebuilding to a folder except if TGZ // If the DatItem is a Disk, force rebuilding to a folder except if TGZ
if (datItem.ItemType == ItemType.Disk && outputFormat != OutputFormat.TorrentGzip) if (datItem.ItemType == ItemType.Disk && outputFormat != OutputFormat.TorrentGzip)
@@ -4152,14 +4152,12 @@ namespace SabreTools.Library.DatFiles
try try
{ {
File.Copy(file, outDir); File.Copy(file, outDir);
rebuilt &= true; return true;
} }
catch catch
{ {
rebuilt = false; return false;
} }
return rebuilt;
} }
// Get a generic stream for the file // Get a generic stream for the file
@@ -4237,14 +4235,12 @@ namespace SabreTools.Library.DatFiles
try try
{ {
File.Copy(file, outDir); File.Copy(file, outDir);
rebuilt &= true; return true;
} }
catch catch
{ {
rebuilt = false; return false;
} }
return rebuilt;
} }
// Get a generic stream for the file // Get a generic stream for the file

View File

@@ -19,7 +19,7 @@ namespace SabreTools.Library.Data
/// <summary> /// <summary>
/// The current toolset version to be used by all child applications /// The current toolset version to be used by all child applications
/// </summary> /// </summary>
public readonly static string Version = "v1.0.0-" + Assembly.GetExecutingAssembly().GetLinkerTime().ToString("yyyy-MM-dd HH:mm:ss"); public readonly static string Version = "v1.0.0-" + File.GetCreationTime(Assembly.GetExecutingAssembly().Location).ToString("yyyy-MM-dd HH:mm:ss");
public const int HeaderHeight = 3; public const int HeaderHeight = 3;
#region 0-byte file constants #region 0-byte file constants

View File

@@ -231,18 +231,6 @@ namespace SabreTools.Library.Data
LastAccessTimePresent = 1 << 3, LastAccessTimePresent = 1 << 3,
} }
/// <summary>
/// Zipfile special status
/// </summary>
/// <remarks>https://github.com/gjefferyes/RomVault/blob/5a93500001f0d068f32cf77a048950717507f733/ROMVault2/SupportedFiles/ZipEnums.cs</remarks>
[Flags]
public enum ZipStatus
{
None = 0x00,
TorrentZip = 1 << 0,
ExtraData = 1 << 1
}
#endregion #endregion
#region DatFile related #region DatFile related

View File

@@ -21,9 +21,6 @@ namespace SabreTools.Library.Data
private static Logger _logger = null; private static Logger _logger = null;
private static int _maxDegreeOfParallelism = System.Environment.ProcessorCount; private static int _maxDegreeOfParallelism = System.Environment.ProcessorCount;
private static string _exeName = new Uri(Assembly.GetExecutingAssembly().GetName().CodeBase).LocalPath;
private static string _exeDir = Path.GetDirectoryName(_exeName);
private static string _args = string.Join(" ", Environment.GetCommandLineArgs());
#endregion #endregion
@@ -41,11 +38,13 @@ namespace SabreTools.Library.Data
} }
set { _logger = value; } set { _logger = value; }
} }
public static int MaxThreads public static int MaxThreads
{ {
get { return _maxDegreeOfParallelism; } get { return _maxDegreeOfParallelism; }
set { _maxDegreeOfParallelism = value; } set { _maxDegreeOfParallelism = value; }
} }
public static ParallelOptions ParallelOptions public static ParallelOptions ParallelOptions
{ {
get get
@@ -56,26 +55,20 @@ namespace SabreTools.Library.Data
}; };
} }
} }
public static string ExeName public static string ExeName
{ {
get get { return new Uri(Assembly.GetExecutingAssembly().GetName().CodeBase).LocalPath; }
{
return _exeName;
}
} }
public static string ExeDir public static string ExeDir
{ {
get get { return Path.GetDirectoryName(ExeName); }
{
return _exeDir;
}
} }
public static string CommandLineArgs public static string CommandLineArgs
{ {
get get { return string.Join(" ", Environment.GetCommandLineArgs()); }
{
return _args;
}
} }
#endregion #endregion

View File

@@ -171,7 +171,7 @@ namespace Compress.File
public void ZipFileAddDirectory() public void ZipFileAddZeroLengthFile()
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }

View File

@@ -30,7 +30,7 @@ namespace Compress
string ZipFilename { get; } string ZipFilename { get; }
long TimeStamp { get; } long TimeStamp { get; }
void ZipFileAddDirectory(); void ZipFileAddZeroLengthFile();
ZipReturn ZipFileCreate(string newFilename); ZipReturn ZipFileCreate(string newFilename);
ZipReturn ZipFileCloseWriteStream(byte[] crc32); ZipReturn ZipFileCloseWriteStream(byte[] crc32);

View File

@@ -0,0 +1,194 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Interop = Zstandard.Net.ZstandardInterop;
namespace Zstandard.Net
{
/// <summary>
/// A Zstandard dictionary improves the compression ratio and speed on small data dramatically.
/// </summary>
/// <remarks>
/// A Zstandard dictionary is calculated with a high number of small sample data.
/// Please refer to the Zstandard documentation for more details.
/// </remarks>
/// <seealso cref="System.IDisposable" />
public sealed class ZstandardDictionary : IDisposable
{
private byte[] dictionary;
private IntPtr ddict;
private Dictionary<int, IntPtr> cdicts = new Dictionary<int, IntPtr>();
private object lockObject = new object();
private bool isDisposed = false;
/// <summary>
/// Initializes a new instance of the <see cref="ZstandardDictionary"/> class.
/// </summary>
/// <param name="dictionary">The dictionary raw data.</param>
public ZstandardDictionary(byte[] dictionary)
{
this.dictionary = dictionary;
}
/// <summary>
/// Initializes a new instance of the <see cref="ZstandardDictionary"/> class.
/// </summary>
/// <param name="dictionaryPath">The dictionary path.</param>
public ZstandardDictionary(string dictionaryPath)
{
this.dictionary = File.ReadAllBytes(dictionaryPath);
}
/// <summary>
/// Initializes a new instance of the <see cref="ZstandardDictionary"/> class.
/// </summary>
/// <param name="dictionaryStream">The dictionary stream.</param>
public ZstandardDictionary(Stream dictionaryStream)
{
using (var memoryStream = new MemoryStream())
{
dictionaryStream.CopyTo(memoryStream);
this.dictionary = memoryStream.ToArray();
}
}
/// <summary>
/// Finalizes an instance of the <see cref="ZstandardDictionary"/> class.
/// </summary>
~ZstandardDictionary()
{
this.Dispose(false);
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Releases unmanaged resources.
/// </summary>
/// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
private void Dispose(bool dispose)
{
if (this.isDisposed == false)
{
this.isDisposed = true;
if (this.ddict != IntPtr.Zero)
{
Interop.ZSTD_freeDDict(this.ddict);
this.ddict = IntPtr.Zero;
}
foreach (var kv in this.cdicts.ToList())
{
Interop.ZSTD_freeCDict(kv.Value);
this.cdicts.Remove(kv.Key);
}
}
}
/// <summary>
/// Gets the compression dictionary for the specified compression level.
/// </summary>
/// <param name="compressionLevel">The compression level.</param>
/// <returns>
/// The IntPtr to the compression dictionary.
/// </returns>
/// <exception cref="ObjectDisposedException">ZstandardDictionary</exception>
internal IntPtr GetCompressionDictionary(int compressionLevel)
{
if (this.isDisposed)
{
throw new ObjectDisposedException(nameof(ZstandardDictionary));
}
lock (this.lockObject)
{
if (this.cdicts.TryGetValue(compressionLevel, out var cdict) == false)
{
this.cdicts[compressionLevel] = cdict = this.CreateCompressionDictionary(compressionLevel);
}
return cdict;
}
}
/// <summary>
/// Gets the decompression dictionary.
/// </summary>
/// <returns>
/// The IntPtr to the decompression dictionary.
/// </returns>
/// <exception cref="ObjectDisposedException">ZstandardDictionary</exception>
internal IntPtr GetDecompressionDictionary()
{
if (this.isDisposed)
{
throw new ObjectDisposedException(nameof(ZstandardDictionary));
}
lock (this.lockObject)
{
if (this.ddict == IntPtr.Zero)
{
this.ddict = this.CreateDecompressionDictionary();
}
return this.ddict;
}
}
/// <summary>
/// Creates a new compression dictionary.
/// </summary>
/// <param name="compressionLevel">The compression level.</param>
/// <returns>
/// The IntPtr to the compression dictionary.
/// </returns>
private IntPtr CreateCompressionDictionary(int compressionLevel)
{
var alloc = GCHandle.Alloc(this.dictionary, GCHandleType.Pinned);
try
{
var dictBuffer = Marshal.UnsafeAddrOfPinnedArrayElement(this.dictionary, 0);
var dictSize = new UIntPtr((uint)this.dictionary.Length);
return Interop.ZSTD_createCDict(dictBuffer, dictSize, compressionLevel);
}
finally
{
alloc.Free();
}
}
/// <summary>
/// Creates a new decompression dictionary.
/// </summary>
/// <returns>
/// The IntPtr to the decompression dictionary.
/// </returns>
private IntPtr CreateDecompressionDictionary()
{
var alloc = GCHandle.Alloc(this.dictionary, GCHandleType.Pinned);
try
{
var dictBuffer = Marshal.UnsafeAddrOfPinnedArrayElement(this.dictionary, 0);
var dictSize = new UIntPtr((uint)this.dictionary.Length);
return Interop.ZSTD_createDDict(dictBuffer, dictSize);
}
finally
{
alloc.Free();
}
}
}
}

View File

@@ -0,0 +1,143 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
namespace Zstandard.Net
{
internal static class ZstandardInterop
{
static ZstandardInterop()
{
if (Environment.OSVersion.Platform == PlatformID.Win32NT)
{
var root = Path.GetDirectoryName(typeof(ZstandardInterop).Assembly.Location);
var path = Environment.Is64BitProcess ? "x64" : "x86";
var file = Path.Combine(root, path, "libzstd.dll");
LoadLibraryEx(file, IntPtr.Zero, LoadLibraryFlags.LOAD_LIBRARY_SEARCH_APPLICATION_DIR);
}
}
[StructLayout(LayoutKind.Sequential)]
public class Buffer
{
public IntPtr Data = IntPtr.Zero;
public UIntPtr Size = UIntPtr.Zero;
public UIntPtr Position = UIntPtr.Zero;
}
public static void ThrowIfError(UIntPtr code)
{
if (ZSTD_isError(code))
{
var errorPtr = ZSTD_getErrorName(code);
var errorMsg = Marshal.PtrToStringAnsi(errorPtr);
throw new IOException(errorMsg);
}
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
[Flags]
private enum LoadLibraryFlags : uint
{
DONT_RESOLVE_DLL_REFERENCES = 0x00000001,
LOAD_IGNORE_CODE_AUTHZ_LEVEL = 0x00000010,
LOAD_LIBRARY_AS_DATAFILE = 0x00000002,
LOAD_LIBRARY_AS_DATAFILE_EXCLUSIVE = 0x00000040,
LOAD_LIBRARY_AS_IMAGE_RESOURCE = 0x00000020,
LOAD_LIBRARY_SEARCH_APPLICATION_DIR = 0x00000200,
LOAD_LIBRARY_SEARCH_DEFAULT_DIRS = 0x00001000,
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR = 0x00000100,
LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800,
LOAD_LIBRARY_SEARCH_USER_DIRS = 0x00000400,
LOAD_WITH_ALTERED_SEARCH_PATH = 0x00000008
}
[DllImport("kernel32", SetLastError = true)]
private static extern IntPtr LoadLibraryEx(string lpFileName, IntPtr hReservedNull, LoadLibraryFlags dwFlags);
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern uint ZSTD_versionNumber();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern int ZSTD_maxCLevel();
//-----------------------------------------------------------------------------------------
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr ZSTD_createCStream();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_initCStream(IntPtr zcs, int compressionLevel);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_freeCStream(IntPtr zcs);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_CStreamInSize();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_CStreamOutSize();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_compressStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer, [MarshalAs(UnmanagedType.LPStruct)] Buffer inputBuffer);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr ZSTD_createCDict(IntPtr dictBuffer, UIntPtr dictSize, int compressionLevel);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_freeCDict(IntPtr cdict);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_initCStream_usingCDict(IntPtr zcs, IntPtr cdict);
//-----------------------------------------------------------------------------------------
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr ZSTD_createDStream();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_initDStream(IntPtr zds);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_freeDStream(IntPtr zds);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_DStreamInSize();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_DStreamOutSize();
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_decompressStream(IntPtr zds, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer, [MarshalAs(UnmanagedType.LPStruct)] Buffer inputBuffer);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern IntPtr ZSTD_createDDict(IntPtr dictBuffer, UIntPtr dictSize);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_freeDDict(IntPtr ddict);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_initDStream_usingDDict(IntPtr zds, IntPtr ddict);
//-----------------------------------------------------------------------------------------
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_flushStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
public static extern UIntPtr ZSTD_endStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer);
//-----------------------------------------------------------------------------------------
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
internal static extern bool ZSTD_isError(UIntPtr code);
[DllImport("libzstd", CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr ZSTD_getErrorName(UIntPtr code);
}
}

View File

@@ -0,0 +1,398 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Runtime.InteropServices;
using Interop = Zstandard.Net.ZstandardInterop;
namespace Zstandard.Net
{
/// <summary>
/// Provides methods and properties for compressing and decompressing streams by using the Zstandard algorithm.
/// </summary>
public class ZstandardStream : Stream
{
private Stream stream;
private CompressionMode mode;
private Boolean leaveOpen;
private Boolean isClosed = false;
private Boolean isDisposed = false;
private Boolean isInitialized = false;
private IntPtr zstream;
private uint zstreamInputSize;
private uint zstreamOutputSize;
private byte[] data;
private bool dataDepleted = false;
private bool dataSkipRead = false;
private int dataPosition = 0;
private int dataSize = 0;
private long position = 0;
private Interop.Buffer outputBuffer = new Interop.Buffer();
private Interop.Buffer inputBuffer = new Interop.Buffer();
/// <summary>
/// Initializes a new instance of the <see cref="ZstandardStream"/> class by using the specified stream and compression mode, and optionally leaves the stream open.
/// </summary>
/// <param name="stream">The stream to compress.</param>
/// <param name="mode">One of the enumeration values that indicates whether to compress or decompress the stream.</param>
/// <param name="leaveOpen">true to leave the stream open after disposing the <see cref="ZstandardStream"/> object; otherwise, false.</param>
public ZstandardStream(Stream stream, CompressionMode mode, bool leaveOpen = false)
{
this.stream = stream ?? throw new ArgumentNullException(nameof(stream));
this.mode = mode;
this.leaveOpen = leaveOpen;
position = 0;
if (mode == CompressionMode.Compress)
{
this.zstreamInputSize = Interop.ZSTD_CStreamInSize().ToUInt32();
this.zstreamOutputSize = Interop.ZSTD_CStreamOutSize().ToUInt32();
this.zstream = Interop.ZSTD_createCStream();
this.data = new byte[(int)this.zstreamOutputSize];
}
if (mode == CompressionMode.Decompress)
{
this.zstreamInputSize = Interop.ZSTD_DStreamInSize().ToUInt32();
this.zstreamOutputSize = Interop.ZSTD_DStreamOutSize().ToUInt32();
this.zstream = Interop.ZSTD_createDStream();
this.data = new byte[(int)this.zstreamInputSize];
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ZstandardStream"/> class by using the specified stream and compression level, and optionally leaves the stream open.
/// </summary>
/// <param name="stream">The stream to compress.</param>
/// <param name="compressionLevel">The compression level.</param>
/// <param name="leaveOpen">true to leave the stream open after disposing the <see cref="ZstandardStream"/> object; otherwise, false.</param>
public ZstandardStream(Stream stream, int compressionLevel, bool leaveOpen = false) : this(stream, CompressionMode.Compress, leaveOpen)
{
this.CompressionLevel = compressionLevel;
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
/// <summary>
/// The version of the native Zstd library.
/// </summary>
public static Version Version
{
get
{
var version = (int)Interop.ZSTD_versionNumber();
return new Version((version / 10000) % 100, (version / 100) % 100, version % 100);
}
}
/// <summary>
/// The maximum compression level supported by the native Zstd library.
/// </summary>
public static int MaxCompressionLevel
{
get
{
return Interop.ZSTD_maxCLevel();
}
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
/// <summary>
/// Gets or sets the compression level to use, the default is 6.
/// </summary>
/// <remarks>
/// To get the maximum compression level see <see cref="MaxCompressionLevel"/>.
/// </remarks>
public int CompressionLevel { get; set; } = 6;
/// <summary>
/// Gets or sets the compression dictionary tp use, the default is null.
/// </summary>
/// <value>
/// The compression dictionary.
/// </value>
public ZstandardDictionary CompressionDictionary { get; set; } = null;
/// <summary>
/// Gets whether the current stream supports reading.
/// </summary>
public override bool CanRead => this.stream.CanRead && this.mode == CompressionMode.Decompress;
/// <summary>
/// Gets whether the current stream supports writing.
/// </summary>
public override bool CanWrite => this.stream.CanWrite && this.mode == CompressionMode.Compress;
/// <summary>
/// Gets whether the current stream supports seeking.
/// </summary>
public override bool CanSeek => false;
/// <summary>
/// Gets the length in bytes of the stream.
/// </summary>
public override long Length => throw new NotSupportedException();
/// <summary>
/// Gets or sets the position within the current stream.
/// </summary>
public override long Position
{
get => position;
set => throw new NotSupportedException();
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (this.isDisposed == false)
{
if (!this.isClosed) ReleaseResources(flushStream: false);
this.isDisposed = true;
this.data = null;
}
}
public override void Close()
{
if (this.isClosed) return;
try
{
ReleaseResources(flushStream: true);
}
finally
{
this.isClosed = true;
base.Close();
}
}
private void ReleaseResources(bool flushStream)
{
if (this.mode == CompressionMode.Compress)
{
try
{
if (flushStream)
{
this.ProcessStream((zcs, buffer) => Interop.ThrowIfError(Interop.ZSTD_flushStream(zcs, buffer)));
this.ProcessStream((zcs, buffer) => Interop.ThrowIfError(Interop.ZSTD_endStream(zcs, buffer)));
this.stream.Flush();
}
}
finally
{
Interop.ZSTD_freeCStream(this.zstream);
if (!this.leaveOpen) this.stream.Close();
}
}
else if (this.mode == CompressionMode.Decompress)
{
Interop.ZSTD_freeDStream(this.zstream);
if (!this.leaveOpen) this.stream.Close();
}
}
public override void Flush()
{
if (this.mode == CompressionMode.Compress)
{
this.ProcessStream((zcs, buffer) => Interop.ThrowIfError(Interop.ZSTD_flushStream(zcs, buffer)));
this.stream.Flush();
}
}
public override int Read(byte[] buffer, int offset, int count)
{
if (this.CanRead == false) throw new NotSupportedException();
// prevent the buffers from being moved around by the garbage collector
var alloc1 = GCHandle.Alloc(buffer, GCHandleType.Pinned);
var alloc2 = GCHandle.Alloc(this.data, GCHandleType.Pinned);
try
{
var length = 0;
if (this.isInitialized == false)
{
this.isInitialized = true;
var result = this.CompressionDictionary == null
? Interop.ZSTD_initDStream(this.zstream)
: Interop.ZSTD_initDStream_usingDDict(this.zstream, this.CompressionDictionary.GetDecompressionDictionary());
}
while (count > 0)
{
var inputSize = this.dataSize - this.dataPosition;
// read data from input stream
if (inputSize <= 0 && !this.dataDepleted && !this.dataSkipRead)
{
this.dataSize = this.stream.Read(this.data, 0, (int)this.zstreamInputSize);
this.dataDepleted = this.dataSize <= 0;
this.dataPosition = 0;
inputSize = this.dataDepleted ? 0 : this.dataSize;
// skip stream.Read until the internal buffer is depleted
// avoids a Read timeout for applications that know the exact number of bytes in the stream
this.dataSkipRead = true;
}
// configure the inputBuffer
this.inputBuffer.Data = inputSize <= 0 ? IntPtr.Zero : Marshal.UnsafeAddrOfPinnedArrayElement(this.data, this.dataPosition);
this.inputBuffer.Size = inputSize <= 0 ? UIntPtr.Zero : new UIntPtr((uint)inputSize);
this.inputBuffer.Position = UIntPtr.Zero;
// configure the outputBuffer
this.outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, offset);
this.outputBuffer.Size = new UIntPtr((uint)count);
this.outputBuffer.Position = UIntPtr.Zero;
// decompress inputBuffer to outputBuffer
Interop.ThrowIfError(Interop.ZSTD_decompressStream(this.zstream, this.outputBuffer, this.inputBuffer));
// calculate progress in outputBuffer
var outputBufferPosition = (int)this.outputBuffer.Position.ToUInt32();
if (outputBufferPosition == 0)
{
// the internal buffer is depleted, we're either done
if (this.dataDepleted) break;
// or we need more bytes
this.dataSkipRead = false;
}
length += outputBufferPosition;
offset += outputBufferPosition;
count -= outputBufferPosition;
// calculate progress in inputBuffer
var inputBufferPosition = (int)inputBuffer.Position.ToUInt32();
this.dataPosition += inputBufferPosition;
}
position += length;
return length;
}
finally
{
alloc1.Free();
alloc2.Free();
}
}
public override void Write(byte[] buffer, int offset, int count)
{
if (this.CanWrite == false) throw new NotSupportedException();
// prevent the buffers from being moved around by the garbage collector
var alloc1 = GCHandle.Alloc(buffer, GCHandleType.Pinned);
var alloc2 = GCHandle.Alloc(this.data, GCHandleType.Pinned);
try
{
if (this.isInitialized == false)
{
this.isInitialized = true;
var result = this.CompressionDictionary == null
? Interop.ZSTD_initCStream(this.zstream, this.CompressionLevel)
: Interop.ZSTD_initCStream_usingCDict(this.zstream, this.CompressionDictionary.GetCompressionDictionary(this.CompressionLevel));
Interop.ThrowIfError(result);
}
while (count > 0)
{
var inputSize = Math.Min((uint)count, this.zstreamInputSize);
// configure the outputBuffer
this.outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(this.data, 0);
this.outputBuffer.Size = new UIntPtr(this.zstreamOutputSize);
this.outputBuffer.Position = UIntPtr.Zero;
// configure the inputBuffer
this.inputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, offset);
this.inputBuffer.Size = new UIntPtr((uint)inputSize);
this.inputBuffer.Position = UIntPtr.Zero;
// compress inputBuffer to outputBuffer
Interop.ThrowIfError(Interop.ZSTD_compressStream(this.zstream, this.outputBuffer, this.inputBuffer));
// write data to output stream
var outputBufferPosition = (int)this.outputBuffer.Position.ToUInt32();
this.stream.Write(this.data, 0, outputBufferPosition);
// calculate progress in inputBuffer
var inputBufferPosition = (int)this.inputBuffer.Position.ToUInt32();
offset += inputBufferPosition;
count -= inputBufferPosition;
}
}
finally
{
alloc1.Free();
alloc2.Free();
}
}
public override long Seek(long offset, SeekOrigin origin)
{
if (origin != SeekOrigin.Current)
throw new NotImplementedException();
byte[] tmpBuff = new byte[1024];
long sizeToGo = offset;
while (sizeToGo > 0)
{
int sizenow = sizeToGo > 1024 ? 1024 : (int)sizeToGo;
Read(tmpBuff, 0, sizenow);
sizeToGo -= sizenow;
}
position += offset;
return offset;
}
public override void SetLength(long value)
{
throw new NotImplementedException();
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
private void ProcessStream(Action<IntPtr, Interop.Buffer> outputAction)
{
var alloc = GCHandle.Alloc(this.data, GCHandleType.Pinned);
try
{
this.outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(this.data, 0);
this.outputBuffer.Size = new UIntPtr(this.zstreamOutputSize);
this.outputBuffer.Position = UIntPtr.Zero;
outputAction(this.zstream, this.outputBuffer);
var outputBufferPosition = (int)this.outputBuffer.Position.ToUInt32();
this.stream.Write(this.data, 0, outputBufferPosition);
}
finally
{
alloc.Free();
}
}
}
}

View File

@@ -26,18 +26,24 @@ namespace Compress.SevenZip.Filters
int end = offset + count - 5; int end = offset + count - 5;
int i; int i;
for (i = offset; i <= end; ++i) { for (i = offset; i <= end; ++i)
{
if ((buffer[i] & 0xFE) != 0xE8) if ((buffer[i] & 0xFE) != 0xE8)
continue; continue;
prevPos = i - prevPos; prevPos = i - prevPos;
if ((prevPos & ~3) != 0) { // (unsigned)prevPos > 3 if ((prevPos & ~3) != 0)
{ // (unsigned)prevPos > 3
prevMask = 0; prevMask = 0;
} else { }
else
{
prevMask = (prevMask << (prevPos - 1)) & 7; prevMask = (prevMask << (prevPos - 1)) & 7;
if (prevMask != 0) { if (prevMask != 0)
{
if (!MASK_TO_ALLOWED_STATUS[prevMask] || test86MSByte( if (!MASK_TO_ALLOWED_STATUS[prevMask] || test86MSByte(
buffer[i + 4 - MASK_TO_BIT_NUMBER[prevMask]])) { buffer[i + 4 - MASK_TO_BIT_NUMBER[prevMask]]))
{
prevPos = i; prevPos = i;
prevMask = (prevMask << 1) | 1; prevMask = (prevMask << 1) | 1;
continue; continue;
@@ -47,13 +53,15 @@ namespace Compress.SevenZip.Filters
prevPos = i; prevPos = i;
if (test86MSByte(buffer[i + 4])) { if (test86MSByte(buffer[i + 4]))
{
int src = buffer[i + 1] int src = buffer[i + 1]
| (buffer[i + 2] << 8) | (buffer[i + 2] << 8)
| (buffer[i + 3] << 16) | (buffer[i + 3] << 16)
| (buffer[i + 4] << 24); | (buffer[i + 4] << 24);
int dest; int dest;
while (true) { while (true)
{
if (isEncoder) if (isEncoder)
dest = src + (pos + i - offset); dest = src + (pos + i - offset);
else else
@@ -74,7 +82,9 @@ namespace Compress.SevenZip.Filters
buffer[i + 3] = (byte)(dest >> 16); buffer[i + 3] = (byte)(dest >> 16);
buffer[i + 4] = (byte)(~(((dest >> 24) & 1) - 1)); buffer[i + 4] = (byte)(~(((dest >> 24) & 1) - 1));
i += 4; i += 4;
} else { }
else
{
prevMask = (prevMask << 1) | 1; prevMask = (prevMask << 1) | 1;
} }
} }

View File

@@ -1,21 +1,38 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO; using System.IO;
using System.Text; using System.Text;
using Compress.SevenZip.Compress.BZip2;
using Compress.SevenZip.Compress.LZMA;
using Compress.SevenZip.Compress.PPmd;
using Compress.SevenZip.Filters;
using Compress.SevenZip.Structure; using Compress.SevenZip.Structure;
using Compress.Utils;
using FileInfo = RVIO.FileInfo; using FileInfo = RVIO.FileInfo;
using FileStream = RVIO.FileStream;
namespace Compress.SevenZip namespace Compress.SevenZip
{ {
public class SevenZ : ICompress public partial class SevenZ : ICompress
{ {
public static bool supportZstd
{
get;
private set;
}
public static void TestForZstd()
{
supportZstd = RVIO.File.Exists("libzstd.dll");
}
private class LocalFile
{
public string FileName;
public ulong UncompressedSize;
public bool IsDirectory;
public byte[] CRC;
public int StreamIndex;
public ulong StreamOffset;
public ZipReturn FileStatus = ZipReturn.ZipUntested;
}
private List<LocalFile> _localFiles = new List<LocalFile>(); private List<LocalFile> _localFiles = new List<LocalFile>();
private FileInfo _zipFileInfo; private FileInfo _zipFileInfo;
@@ -105,186 +122,13 @@ namespace Compress.SevenZip
} }
public void ZipFileAddDirectory(string filename)
{
string fName = filename;
if (fName.Substring(fName.Length - 1, 1) == @"/")
fName = fName.Substring(0, fName.Length - 1);
LocalFile lf = new LocalFile
{
FileName = fName,
UncompressedSize = 0,
IsDirectory = true,
StreamOffset = 0
};
_localFiles.Add(lf);
}
private class LocalFile
{
public string FileName;
public ulong UncompressedSize;
public bool IsDirectory;
public byte[] CRC;
public int StreamIndex;
public ulong StreamOffset;
public ZipReturn FileStatus = ZipReturn.ZipUntested;
}
#region open 7z files
public ZipReturn ZipFileOpen(string filename, long timestamp, bool readHeaders)
{
ZipFileClose();
Debug.WriteLine(filename);
#region open file stream
try
{
if (!RVIO.File.Exists(filename))
{
ZipFileClose();
return ZipReturn.ZipErrorFileNotFound;
}
_zipFileInfo = new FileInfo(filename);
if ((timestamp != -1) && (_zipFileInfo.LastWriteTime != timestamp))
{
ZipFileClose();
return ZipReturn.ZipErrorTimeStamp;
}
int errorCode = FileStream.OpenFileRead(filename, out _zipFs);
if (errorCode != 0)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
}
catch (PathTooLongException)
{
ZipFileClose();
return ZipReturn.ZipFileNameToLong;
}
catch (IOException)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
#endregion
ZipOpen = ZipOpenType.OpenRead;
ZipStatus = ZipStatus.None;
return ZipFileReadHeaders();
}
public ZipReturn ZipFileOpen(Stream inStream)
{
ZipFileClose();
_zipFileInfo = null;
_zipFs = inStream;
ZipOpen = ZipOpenType.OpenRead;
ZipStatus = ZipStatus.None;
return ZipFileReadHeaders();
}
private ZipReturn ZipFileReadHeaders()
{
try
{
SignatureHeader signatureHeader = new SignatureHeader();
if (!signatureHeader.Read(_zipFs))
{
return ZipReturn.ZipSignatureError;
}
_baseOffset = _zipFs.Position;
//_zipFs.Seek(_baseOffset + (long)signatureHeader.NextHeaderOffset, SeekOrigin.Begin);
//byte[] mainHeader = new byte[signatureHeader.NextHeaderSize];
//_zipFs.Read(mainHeader, 0, (int)signatureHeader.NextHeaderSize);
//if (!CRC.VerifyDigest(signatureHeader.NextHeaderCRC, mainHeader, 0, (uint)signatureHeader.NextHeaderSize))
// return ZipReturn.Zip64EndOfCentralDirError;
if (signatureHeader.NextHeaderSize != 0)
{
_zipFs.Seek(_baseOffset + (long)signatureHeader.NextHeaderOffset, SeekOrigin.Begin);
ZipReturn zr = Header.ReadHeaderOrPackedHeader(_zipFs, _baseOffset, out _header);
if (zr != ZipReturn.ZipGood)
{
return zr;
}
}
_zipFs.Seek(_baseOffset + (long)(signatureHeader.NextHeaderOffset + signatureHeader.NextHeaderSize), SeekOrigin.Begin);
ZipStatus = ZipStatus.None;
ZipStatus |= IsRomVault7Z() ? ZipStatus.TrrntZip : ZipStatus.None;
ZipStatus |= Istorrent7Z() ? ZipStatus.Trrnt7Zip : ZipStatus.None;
PopulateLocalFiles(out _localFiles);
return ZipReturn.ZipGood;
}
catch
{
ZipFileClose();
return ZipReturn.ZipErrorReadingFile;
}
}
private void PopulateLocalFiles(out List<LocalFile> localFiles)
{
int emptyFileIndex = 0;
int folderIndex = 0;
int unpackedStreamsIndex = 0;
ulong streamOffset = 0;
localFiles = new List<LocalFile>();
if (_header == null)
return;
for (int i = 0; i < _header.FileInfo.Names.Length; i++)
{
LocalFile lf = new LocalFile { FileName = _header.FileInfo.Names[i] };
if ((_header.FileInfo.EmptyStreamFlags == null) || !_header.FileInfo.EmptyStreamFlags[i])
{
lf.StreamIndex = folderIndex;
lf.StreamOffset = streamOffset;
lf.UncompressedSize = _header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo[unpackedStreamsIndex].UnpackedSize;
lf.CRC = Util.uinttobytes(_header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo[unpackedStreamsIndex].Crc);
streamOffset += lf.UncompressedSize;
unpackedStreamsIndex++;
if (unpackedStreamsIndex >= _header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo.Length)
{
folderIndex++;
unpackedStreamsIndex = 0;
streamOffset = 0;
}
}
else
{
lf.UncompressedSize = 0;
lf.CRC = new byte[] { 0, 0, 0, 0 };
lf.IsDirectory = (_header.FileInfo.EmptyFileFlags == null) || !_header.FileInfo.EmptyFileFlags[emptyFileIndex++];
if (lf.IsDirectory)
{
if (lf.FileName.Substring(lf.FileName.Length - 1, 1) != "/")
{
lf.FileName += "/";
}
}
}
localFiles.Add(lf);
}
}
public void ZipFileClose() public void ZipFileClose()
@@ -312,6 +156,7 @@ namespace Compress.SevenZip
ZipOpen = ZipOpenType.Closed; ZipOpen = ZipOpenType.Closed;
} }
private Header _header; private Header _header;
public StringBuilder HeaderReport() public StringBuilder HeaderReport()
@@ -328,755 +173,5 @@ namespace Compress.SevenZip
return sb; return sb;
} }
// not finalized yet, so do not use
private void WriteRomVault7Zip(BinaryWriter bw, ulong headerPos, ulong headerLength, uint headerCRC)
{
const string sig = "RomVault7Z01";
byte[] RV7Zid = Util.Enc.GetBytes(sig);
// RomVault 7Zip torrent header
// 12 bytes : RomVault7Zip
// 4 bytes : HeaderCRC
// 8 bytes : HeaderPos
// 8 bytes : HeaderLength
bw.Write(RV7Zid);
bw.Write(headerCRC);
bw.Write(headerPos);
bw.Write(headerLength);
ZipStatus = ZipStatus.TrrntZip;
}
private bool IsRomVault7Z()
{
long length = _zipFs.Length;
if (length < 32)
{
return false;
}
_zipFs.Seek(length - 32, SeekOrigin.Begin);
const string sig = "RomVault7Z01";
byte[] rv7Zid = Util.Enc.GetBytes(sig);
byte[] header = new byte[12];
_zipFs.Read(header, 0, 12);
for (int i = 0; i < 12; i++)
{
if (header[i] != rv7Zid[i])
{
return false;
}
}
uint headerCRC;
ulong headerOffset;
ulong headerSize;
using (BinaryReader br = new BinaryReader(_zipFs, Encoding.UTF8, true))
{
headerCRC = br.ReadUInt32();
headerOffset = br.ReadUInt64();
headerSize = br.ReadUInt64();
}
if ((ulong)length < headerOffset)
{
return false;
}
_zipFs.Seek((long)headerOffset, SeekOrigin.Begin);
byte[] mainHeader = new byte[headerSize];
int bytesread = _zipFs.Read(mainHeader, 0, (int)headerSize);
return ((ulong)bytesread == headerSize) &&
Utils.CRC.VerifyDigest(headerCRC, mainHeader, 0, (uint)headerSize);
}
private bool Istorrent7Z()
{
const int crcsz = 128;
const int t7ZsigSize = 16 + 1 + 9 + 4 + 4;
byte[] kSignature = { (byte)'7', (byte)'z', 0xBC, 0xAF, 0x27, 0x1C };
int kSignatureSize = kSignature.Length;
const string sig = "\xa9\x9f\xd1\x57\x08\xa9\xd7\xea\x29\x64\xb2\x36\x1b\x83\x52\x33\x01torrent7z_0.9beta";
byte[] t7Zid = Util.Enc.GetBytes(sig);
int t7ZidSize = t7Zid.Length;
const int tmpbufsize = 256 + t7ZsigSize + 8 + 4;
byte[] buffer = new byte[tmpbufsize];
// read fist 128 bytes, pad with zeros if less bytes
int bufferPos = 0;
_zipFs.Seek(0, SeekOrigin.Begin);
int ar = _zipFs.Read(buffer, bufferPos, crcsz);
if (ar < crcsz)
{
Util.memset(buffer, bufferPos + ar, 0, crcsz - ar);
}
bufferPos = crcsz;
long foffs = _zipFs.Length;
int endReadLength = crcsz + t7ZsigSize + 4;
foffs = foffs < endReadLength ? 0 : foffs - endReadLength;
_zipFs.Seek(foffs, SeekOrigin.Begin);
ar = _zipFs.Read(buffer, bufferPos, endReadLength);
if (ar < endReadLength)
{
if (ar >= t7ZsigSize + 4)
{
ar -= t7ZsigSize + 4;
}
if (ar < kSignatureSize)
{
ar = kSignatureSize;
}
Util.memset(buffer, bufferPos + ar, 0, crcsz - ar);
Util.memcpyr(buffer, crcsz * 2 + 8, buffer, bufferPos + ar, t7ZsigSize + 4);
}
else
{
Util.memcpyr(buffer, crcsz * 2 + 8, buffer, crcsz * 2, t7ZsigSize + 4);
}
foffs = _zipFs.Length;
foffs -= t7ZsigSize + 4;
//memcpy(buffer, crcsz * 2, &foffs, 8);
buffer[crcsz * 2 + 0] = (byte)((foffs >> 0) & 0xff);
buffer[crcsz * 2 + 1] = (byte)((foffs >> 8) & 0xff);
buffer[crcsz * 2 + 2] = (byte)((foffs >> 16) & 0xff);
buffer[crcsz * 2 + 3] = (byte)((foffs >> 24) & 0xff);
buffer[crcsz * 2 + 4] = 0;
buffer[crcsz * 2 + 5] = 0;
buffer[crcsz * 2 + 6] = 0;
buffer[crcsz * 2 + 7] = 0;
if (Util.memcmp(buffer, 0, kSignature, kSignatureSize))
{
t7Zid[16] = buffer[crcsz * 2 + 4 + 8 + 16];
if (Util.memcmp(buffer, crcsz * 2 + 4 + 8, t7Zid, t7ZidSize))
{
uint inCrc32 = (uint)(buffer[crcsz * 2 + 8 + 0] +
(buffer[crcsz * 2 + 8 + 1] << 8) +
(buffer[crcsz * 2 + 8 + 2] << 16) +
(buffer[crcsz * 2 + 8 + 3] << 24));
buffer[crcsz * 2 + 8 + 0] = 0xff;
buffer[crcsz * 2 + 8 + 1] = 0xff;
buffer[crcsz * 2 + 8 + 2] = 0xff;
buffer[crcsz * 2 + 8 + 3] = 0xff;
uint calcCrc32 = Utils.CRC.CalculateDigest(buffer, 0, crcsz * 2 + 8 + t7ZsigSize + 4);
if (inCrc32 == calcCrc32)
{
return true;
}
}
}
return false;
}
#endregion
#region read 7z file
private int _streamIndex = -1;
private Stream _stream;
public ZipReturn ZipFileOpenReadStream(int index, out Stream stream, out ulong unCompressedSize)
{
Debug.WriteLine("Opening File " + _localFiles[index].FileName);
stream = null;
unCompressedSize = 0;
try
{
if (ZipOpen != ZipOpenType.OpenRead)
{
return ZipReturn.ZipErrorGettingDataStream;
}
if (IsDirectory(index))
{
return ZipReturn.ZipTryingToAccessADirectory;
}
unCompressedSize = _localFiles[index].UncompressedSize;
int thisStreamIndex = _localFiles[index].StreamIndex;
ulong streamOffset = _localFiles[index].StreamOffset;
if ((thisStreamIndex == _streamIndex) && (streamOffset >= (ulong)_stream.Position))
{
stream = _stream;
stream.Seek((long)_localFiles[index].StreamOffset - _stream.Position, SeekOrigin.Current);
return ZipReturn.ZipGood;
}
ZipFileCloseReadStream();
_streamIndex = thisStreamIndex;
Folder folder = _header.StreamsInfo.Folders[_streamIndex];
// first make the List of Decompressors streams
int codersNeeded = folder.Coders.Length;
List<InStreamSourceInfo> allInputStreams = new List<InStreamSourceInfo>();
for (int i = 0; i < codersNeeded; i++)
{
folder.Coders[i].DecoderStream = null;
allInputStreams.AddRange(folder.Coders[i].InputStreamsSourceInfo);
}
// now use the binding pairs to links the outputs to the inputs
int bindPairsCount = folder.BindPairs.Length;
for (int i = 0; i < bindPairsCount; i++)
{
allInputStreams[(int)folder.BindPairs[i].InIndex].InStreamSource = InStreamSource.CompStreamOutput;
allInputStreams[(int)folder.BindPairs[i].InIndex].InStreamIndex = folder.BindPairs[i].OutIndex;
folder.Coders[(int)folder.BindPairs[i].OutIndex].OutputUsedInternally = true;
}
// next use the stream indises to connect the remaining input streams from the sourcefile
int packedStreamsCount = folder.PackedStreamIndices.Length;
for (int i = 0; i < packedStreamsCount; i++)
{
ulong packedStreamIndex = (ulong)i + folder.PackedStreamIndexBase;
// create and open the source file stream if needed
if (_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream == null)
{
_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream = CloneStream(_zipFs);
}
_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream.Seek(
_baseOffset + (long)_header.StreamsInfo.PackedStreams[packedStreamIndex].StreamPosition, SeekOrigin.Begin);
allInputStreams[(int)folder.PackedStreamIndices[i]].InStreamSource = InStreamSource.FileStream;
allInputStreams[(int)folder.PackedStreamIndices[i]].InStreamIndex = packedStreamIndex;
}
List<Stream> inputCoders = new List<Stream>();
bool allCodersComplete = false;
while (!allCodersComplete)
{
allCodersComplete = true;
for (int i = 0; i < codersNeeded; i++)
{
Coder coder = folder.Coders[i];
// check is decoder already processed
if (coder.DecoderStream != null)
{
continue;
}
inputCoders.Clear();
for (int j = 0; j < (int)coder.NumInStreams; j++)
{
if (coder.InputStreamsSourceInfo[j].InStreamSource == InStreamSource.FileStream)
{
inputCoders.Add(_header.StreamsInfo.PackedStreams[coder.InputStreamsSourceInfo[j].InStreamIndex].PackedStream);
}
else if (coder.InputStreamsSourceInfo[j].InStreamSource == InStreamSource.CompStreamOutput)
{
if (folder.Coders[coder.InputStreamsSourceInfo[j].InStreamIndex].DecoderStream == null)
{
break;
}
inputCoders.Add(folder.Coders[coder.InputStreamsSourceInfo[j].InStreamIndex].DecoderStream);
}
else
{
// unknown input type so error
return ZipReturn.ZipDecodeError;
}
}
if (inputCoders.Count == (int)coder.NumInStreams)
{
// all inputs streams are available to make the decoder stream
switch (coder.DecoderType)
{
case DecompressType.Stored:
coder.DecoderStream = inputCoders[0];
break;
case DecompressType.Delta:
coder.DecoderStream = new Delta(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.LZMA:
coder.DecoderStream = new LzmaStream(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.LZMA2:
coder.DecoderStream = new LzmaStream(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.PPMd:
coder.DecoderStream = new PpmdStream(new PpmdProperties(folder.Coders[i].Properties), inputCoders[0], false);
break;
case DecompressType.BZip2:
coder.DecoderStream = new CBZip2InputStream(inputCoders[0], false);
break;
case DecompressType.BCJ:
coder.DecoderStream = new BCJFilter(false, inputCoders[0]);
break;
case DecompressType.BCJ2:
coder.DecoderStream = new BCJ2Filter(inputCoders[0], inputCoders[1], inputCoders[2], inputCoders[3]);
break;
default:
return ZipReturn.ZipDecodeError;
}
}
// if skipped a coder need to loop round again
if (coder.DecoderStream == null)
{
allCodersComplete = false;
}
}
}
// find the final output stream and return it.
int outputStream = -1;
for (int i = 0; i < codersNeeded; i++)
{
Coder coder = folder.Coders[i];
if (!coder.OutputUsedInternally)
{
outputStream = i;
}
}
stream = folder.Coders[outputStream].DecoderStream;
stream.Seek((long)_localFiles[index].StreamOffset, SeekOrigin.Current);
_stream = stream;
return ZipReturn.ZipGood;
}
catch (Exception e)
{
return ZipReturn.ZipErrorGettingDataStream;
}
}
private Stream CloneStream(Stream s)
{
switch (s)
{
case System.IO.FileStream _:
int errorCode = FileStream.OpenFileRead(ZipFilename, out Stream streamOut);
return errorCode != 0 ? null : streamOut;
case MemoryStream memStream:
long pos = memStream.Position;
memStream.Position = 0;
byte[] newStream = new byte[memStream.Length];
memStream.Read(newStream, 0, (int)memStream.Length);
MemoryStream ret = new MemoryStream(newStream, false);
memStream.Position = pos;
return ret;
}
return null;
}
public ZipReturn ZipFileCloseReadStream()
{
if (_streamIndex != -1)
{
Folder folder = _header.StreamsInfo.Folders[_streamIndex];
foreach (Coder c in folder.Coders)
{
Stream ds = c?.DecoderStream;
if (ds == null)
{
continue;
}
ds.Close();
ds.Dispose();
c.DecoderStream = null;
}
}
_streamIndex = -1;
if (_header?.StreamsInfo != null)
{
foreach (PackedStreamInfo psi in _header.StreamsInfo.PackedStreams)
{
if (psi?.PackedStream == null)
{
continue;
}
psi.PackedStream.Close();
psi.PackedStream.Dispose();
psi.PackedStream = null;
}
}
return ZipReturn.ZipGood;
}
#endregion
#region write 7z File
private LzmaStream _lzmaStream;
private ulong _packStreamStart;
private ulong _packStreamSize;
private ulong _unpackedStreamSize;
private byte[] _codeMSbytes;
public void ZipFileAddDirectory()
{
// do nothing here for 7zip
}
public ZipReturn ZipFileCreate(string newFilename)
{
return ZipFileCreate(newFilename, true);
}
public ZipReturn ZipFileCreateFromUncompressedSize(string newFilename, ulong unCompressedSize)
{
return ZipFileCreate(newFilename,true, GetDictionarySizeFromUncompressedSize(unCompressedSize));
}
public ZipReturn ZipFileCreate(string newFilename, bool compressOutput, int dictionarySize = 1 << 24, int numFastBytes = 64)
{
if (ZipOpen != ZipOpenType.Closed)
{
return ZipReturn.ZipFileAlreadyOpen;
}
DirUtil.CreateDirForFile(newFilename);
_zipFileInfo = new FileInfo(newFilename);
int errorCode = FileStream.OpenFileWrite(newFilename, out _zipFs);
if (errorCode != 0)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
ZipOpen = ZipOpenType.OpenWrite;
_signatureHeader = new SignatureHeader();
_header = new Header();
using (BinaryWriter bw = new BinaryWriter(_zipFs, Encoding.UTF8, true))
{
_signatureHeader.Write(bw);
}
_compressed = compressOutput;
_unpackedStreamSize = 0;
if (_compressed)
{
LzmaEncoderProperties ep = new LzmaEncoderProperties(true, dictionarySize, numFastBytes);
_lzmaStream = new LzmaStream(ep, false, _zipFs);
_codeMSbytes = _lzmaStream.Properties;
_packStreamStart = (ulong)_zipFs.Position;
}
return ZipReturn.ZipGood;
}
public ZipReturn ZipFileOpenWriteStream(bool raw, bool trrntzip, string filename, ulong uncompressedSize, ushort compressionMethod, out Stream stream)
{
return ZipFileOpenWriteStream(filename, uncompressedSize, out stream);
}
private ZipReturn ZipFileOpenWriteStream(string filename, ulong uncompressedSize, out Stream stream)
{
LocalFile lf = new LocalFile
{
FileName = filename,
UncompressedSize = uncompressedSize,
StreamOffset = (ulong)(_zipFs.Position - _signatureHeader.BaseOffset)
};
_unpackedStreamSize += uncompressedSize;
_localFiles.Add(lf);
stream = _compressed ? _lzmaStream : _zipFs;
return ZipReturn.ZipGood;
}
public ZipReturn ZipFileCloseWriteStream(byte[] crc32)
{
_localFiles[_localFiles.Count - 1].CRC = new[] { crc32[3], crc32[2], crc32[1], crc32[0] };
return ZipReturn.ZipGood;
}
private void CloseWriting7Zip()
{
if (_compressed)
{
_lzmaStream.Close();
}
_packStreamSize = (ulong)_zipFs.Position - _packStreamStart;
Create7ZStructure();
byte[] newHeaderByte;
using (Stream headerMem = new MemoryStream())
{
using (BinaryWriter headerBw = new BinaryWriter(headerMem, Encoding.UTF8, true))
{
_header.WriteHeader(headerBw);
newHeaderByte = new byte[headerMem.Length];
headerMem.Position = 0;
headerMem.Read(newHeaderByte, 0, newHeaderByte.Length);
}
}
uint mainHeaderCRC = Utils.CRC.CalculateDigest(newHeaderByte, 0, (uint)newHeaderByte.Length);
ulong headerpos = (ulong)_zipFs.Position;
_zipFs.Write(newHeaderByte,0,newHeaderByte.Length);
using (BinaryWriter bw = new BinaryWriter(_zipFs, Encoding.UTF8, true))
{
_signatureHeader.WriteFinal(bw, headerpos, (ulong)newHeaderByte.Length, mainHeaderCRC);
WriteRomVault7Zip(bw, headerpos, (ulong)newHeaderByte.Length, mainHeaderCRC);
}
_zipFs.Flush();
_zipFs.Close();
_zipFs.Dispose();
}
private void Create7ZStructure()
{
int fileCount = _localFiles.Count;
//FileInfo
_header.FileInfo = new Structure.FileInfo
{
Names = new string[fileCount]
};
ulong emptyStreamCount = 0;
ulong emptyFileCount = 0;
for (int i = 0; i < fileCount; i++)
{
_header.FileInfo.Names[i] = _localFiles[i].FileName;
if (_localFiles[i].UncompressedSize != 0)
{
continue;
}
if (!_localFiles[i].IsDirectory)
{
emptyFileCount += 1;
}
emptyStreamCount += 1;
}
ulong outFileCount = (ulong)_localFiles.Count - emptyStreamCount;
_header.FileInfo.EmptyStreamFlags = null;
_header.FileInfo.EmptyFileFlags = null;
_header.FileInfo.Attributes = null;
if (emptyStreamCount > 0)
{
if (emptyStreamCount != emptyFileCount) //then we found directories and need to set the attributes
{
_header.FileInfo.Attributes = new uint[fileCount];
}
if (emptyFileCount > 0)
{
_header.FileInfo.EmptyFileFlags = new bool[emptyStreamCount];
}
emptyStreamCount = 0;
_header.FileInfo.EmptyStreamFlags = new bool[fileCount];
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize != 0)
{
continue;
}
if (_localFiles[i].IsDirectory)
{
if (_header.FileInfo.Attributes != null)
_header.FileInfo.Attributes[i] = 0x10; // set attributes to directory
}
else
{
if (_header.FileInfo.EmptyFileFlags != null)
_header.FileInfo.EmptyFileFlags[emptyStreamCount] = true; // set empty file flag
}
_header.FileInfo.EmptyStreamFlags[i] = true;
emptyStreamCount += 1;
}
}
//StreamsInfo
_header.StreamsInfo = new StreamsInfo { PackPosition = 0 };
//StreamsInfo.PackedStreamsInfo
if (_compressed)
{
_header.StreamsInfo.PackedStreams = new PackedStreamInfo[1];
_header.StreamsInfo.PackedStreams[0] = new PackedStreamInfo { PackedSize = _packStreamSize };
}
else
{
_header.StreamsInfo.PackedStreams = new PackedStreamInfo[outFileCount];
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
_header.StreamsInfo.PackedStreams[fileIndex++] = new PackedStreamInfo { PackedSize = _localFiles[i].UncompressedSize };
}
}
//StreamsInfo.PackedStreamsInfo, no CRC or StreamPosition required
if (_compressed)
{
//StreamsInfo.Folders
_header.StreamsInfo.Folders = new Folder[1];
Folder folder = new Folder { Coders = new Coder[1] };
//StreamsInfo.Folders.Coder
// flags 0x23
folder.Coders[0] = new Coder
{
Method = new byte[] { 3, 1, 1 },
NumInStreams = 1,
NumOutStreams = 1,
Properties = _codeMSbytes
};
folder.BindPairs = null;
folder.PackedStreamIndices = new[] { (ulong)0 };
folder.UnpackedStreamSizes = new[] { _unpackedStreamSize };
folder.UnpackCRC = null;
folder.UnpackedStreamInfo = new UnpackedStreamInfo[outFileCount];
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
UnpackedStreamInfo unpackedStreamInfo = new UnpackedStreamInfo
{
UnpackedSize = _localFiles[i].UncompressedSize,
Crc = Util.bytestouint(_localFiles[i].CRC)
};
folder.UnpackedStreamInfo[fileIndex++] = unpackedStreamInfo;
}
_header.StreamsInfo.Folders[0] = folder;
}
else
{
_header.StreamsInfo.Folders = new Folder[outFileCount];
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
Folder folder = new Folder { Coders = new Coder[1] };
//StreamsInfo.Folders.Coder
// flags 0x01
folder.Coders[0] = new Coder
{
Method = new byte[] { 0 },
NumInStreams = 1,
NumOutStreams = 1,
Properties = null
};
folder.BindPairs = null;
folder.PackedStreamIndices = new[] { (ulong)i };
folder.UnpackedStreamSizes = new[] { _localFiles[i].UncompressedSize };
folder.UnpackCRC = null;
folder.UnpackedStreamInfo = new UnpackedStreamInfo[1];
UnpackedStreamInfo unpackedStreamInfo = new UnpackedStreamInfo
{
UnpackedSize = _localFiles[i].UncompressedSize,
Crc = Util.bytestouint(_localFiles[i].CRC)
};
folder.UnpackedStreamInfo[0] = unpackedStreamInfo;
_header.StreamsInfo.Folders[fileIndex++] = folder;
}
}
}
#endregion
private static readonly int[] DictionarySizes =
{
0x10000,
0x18000,
0x20000,
0x30000,
0x40000,
0x60000,
0x80000,
0xc0000,
0x100000,
0x180000,
0x200000,
0x300000,
0x400000,
0x600000,
0x800000,
0xc00000,
0x1000000,
0x1800000,
0x2000000,
0x3000000,
0x4000000,
0x6000000
};
private static int GetDictionarySizeFromUncompressedSize(ulong unCompressedSize)
{
foreach (int v in DictionarySizes)
{
if ((ulong)v >= unCompressedSize)
return v;
}
return DictionarySizes[DictionarySizes.Length - 1];
}
} }
} }

View File

@@ -0,0 +1,170 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using Compress.SevenZip.Structure;
using Compress.Utils;
using FileInfo = RVIO.FileInfo;
using FileStream = RVIO.FileStream;
namespace Compress.SevenZip
{
public partial class SevenZ
{
public ZipReturn ZipFileOpen(string filename, long timestamp, bool readHeaders)
{
ZipFileClose();
Debug.WriteLine(filename);
#region open file stream
try
{
if (!RVIO.File.Exists(filename))
{
ZipFileClose();
return ZipReturn.ZipErrorFileNotFound;
}
_zipFileInfo = new FileInfo(filename);
if ((timestamp != -1) && (_zipFileInfo.LastWriteTime != timestamp))
{
ZipFileClose();
return ZipReturn.ZipErrorTimeStamp;
}
int errorCode = FileStream.OpenFileRead(filename, out _zipFs);
if (errorCode != 0)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
}
catch (PathTooLongException)
{
ZipFileClose();
return ZipReturn.ZipFileNameToLong;
}
catch (IOException)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
#endregion
ZipOpen = ZipOpenType.OpenRead;
ZipStatus = ZipStatus.None;
return ZipFileReadHeaders();
}
public ZipReturn ZipFileOpen(Stream inStream)
{
ZipFileClose();
_zipFileInfo = null;
_zipFs = inStream;
ZipOpen = ZipOpenType.OpenRead;
ZipStatus = ZipStatus.None;
return ZipFileReadHeaders();
}
private ZipReturn ZipFileReadHeaders()
{
try
{
SignatureHeader signatureHeader = new SignatureHeader();
if (!signatureHeader.Read(_zipFs))
{
return ZipReturn.ZipSignatureError;
}
_baseOffset = _zipFs.Position;
_zipFs.Seek(_baseOffset + (long)signatureHeader.NextHeaderOffset, SeekOrigin.Begin);
byte[] mainHeader = new byte[signatureHeader.NextHeaderSize];
_zipFs.Read(mainHeader, 0, (int)signatureHeader.NextHeaderSize);
if (!CRC.VerifyDigest(signatureHeader.NextHeaderCRC, mainHeader, 0, (uint)signatureHeader.NextHeaderSize))
return ZipReturn.Zip64EndOfCentralDirError;
if (signatureHeader.NextHeaderSize != 0)
{
_zipFs.Seek(_baseOffset + (long)signatureHeader.NextHeaderOffset, SeekOrigin.Begin);
ZipReturn zr = Header.ReadHeaderOrPackedHeader(_zipFs, _baseOffset, out _header);
if (zr != ZipReturn.ZipGood)
{
return zr;
}
}
ZipStatus = ZipStatus.None;
ZipStatus |= IsRomVault7Z(_baseOffset, signatureHeader.NextHeaderOffset, signatureHeader.NextHeaderSize, signatureHeader.NextHeaderCRC) ? ZipStatus.TrrntZip : ZipStatus.None;
_zipFs.Seek(_baseOffset + (long)(signatureHeader.NextHeaderOffset + signatureHeader.NextHeaderSize), SeekOrigin.Begin);
ZipStatus |= Istorrent7Z() ? ZipStatus.Trrnt7Zip : ZipStatus.None;
PopulateLocalFiles(out _localFiles);
return ZipReturn.ZipGood;
}
catch
{
ZipFileClose();
return ZipReturn.ZipErrorReadingFile;
}
}
private void PopulateLocalFiles(out List<LocalFile> localFiles)
{
int emptyFileIndex = 0;
int folderIndex = 0;
int unpackedStreamsIndex = 0;
ulong streamOffset = 0;
localFiles = new List<LocalFile>();
if (_header == null)
return;
for (int i = 0; i < _header.FileInfo.Names.Length; i++)
{
LocalFile lf = new LocalFile { FileName = _header.FileInfo.Names[i] };
if ((_header.FileInfo.EmptyStreamFlags == null) || !_header.FileInfo.EmptyStreamFlags[i])
{
lf.StreamIndex = folderIndex;
lf.StreamOffset = streamOffset;
lf.UncompressedSize = _header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo[unpackedStreamsIndex].UnpackedSize;
lf.CRC = Util.uinttobytes(_header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo[unpackedStreamsIndex].Crc);
streamOffset += lf.UncompressedSize;
unpackedStreamsIndex++;
if (unpackedStreamsIndex >= _header.StreamsInfo.Folders[folderIndex].UnpackedStreamInfo.Length)
{
folderIndex++;
unpackedStreamsIndex = 0;
streamOffset = 0;
}
}
else
{
lf.UncompressedSize = 0;
lf.CRC = new byte[] { 0, 0, 0, 0 };
lf.IsDirectory = (_header.FileInfo.EmptyFileFlags == null) || !_header.FileInfo.EmptyFileFlags[emptyFileIndex++];
if (lf.IsDirectory)
{
if (lf.FileName.Substring(lf.FileName.Length - 1, 1) != "/")
{
lf.FileName += "/";
}
}
}
localFiles.Add(lf);
}
}
}
}

View File

@@ -0,0 +1,260 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using Compress.SevenZip.Compress.BZip2;
using Compress.SevenZip.Compress.LZMA;
using Compress.SevenZip.Compress.PPmd;
using Compress.SevenZip.Filters;
using Compress.SevenZip.Structure;
using Zstandard.Net;
using FileStream = RVIO.FileStream;
namespace Compress.SevenZip
{
public partial class SevenZ
{
private int _streamIndex = -1;
private Stream _stream;
public ZipReturn ZipFileOpenReadStream(int index, out Stream stream, out ulong unCompressedSize)
{
Debug.WriteLine("Opening File " + _localFiles[index].FileName);
stream = null;
unCompressedSize = 0;
try
{
if (ZipOpen != ZipOpenType.OpenRead)
{
return ZipReturn.ZipErrorGettingDataStream;
}
if (IsDirectory(index))
{
return ZipReturn.ZipTryingToAccessADirectory;
}
unCompressedSize = _localFiles[index].UncompressedSize;
int thisStreamIndex = _localFiles[index].StreamIndex;
ulong streamOffset = _localFiles[index].StreamOffset;
if ((thisStreamIndex == _streamIndex) && (streamOffset >= (ulong)_stream.Position))
{
stream = _stream;
stream.Seek((long)_localFiles[index].StreamOffset - _stream.Position, SeekOrigin.Current);
return ZipReturn.ZipGood;
}
ZipFileCloseReadStream();
_streamIndex = thisStreamIndex;
Folder folder = _header.StreamsInfo.Folders[_streamIndex];
// first make the List of Decompressors streams
int codersNeeded = folder.Coders.Length;
List<InStreamSourceInfo> allInputStreams = new List<InStreamSourceInfo>();
for (int i = 0; i < codersNeeded; i++)
{
folder.Coders[i].DecoderStream = null;
allInputStreams.AddRange(folder.Coders[i].InputStreamsSourceInfo);
}
// now use the binding pairs to links the outputs to the inputs
int bindPairsCount = folder.BindPairs.Length;
for (int i = 0; i < bindPairsCount; i++)
{
allInputStreams[(int)folder.BindPairs[i].InIndex].InStreamSource = InStreamSource.CompStreamOutput;
allInputStreams[(int)folder.BindPairs[i].InIndex].InStreamIndex = folder.BindPairs[i].OutIndex;
folder.Coders[(int)folder.BindPairs[i].OutIndex].OutputUsedInternally = true;
}
// next use the stream indises to connect the remaining input streams from the sourcefile
int packedStreamsCount = folder.PackedStreamIndices.Length;
for (int i = 0; i < packedStreamsCount; i++)
{
ulong packedStreamIndex = (ulong)i + folder.PackedStreamIndexBase;
// create and open the source file stream if needed
if (_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream == null)
{
_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream = CloneStream(_zipFs);
}
_header.StreamsInfo.PackedStreams[packedStreamIndex].PackedStream.Seek(
_baseOffset + (long)_header.StreamsInfo.PackedStreams[packedStreamIndex].StreamPosition, SeekOrigin.Begin);
allInputStreams[(int)folder.PackedStreamIndices[i]].InStreamSource = InStreamSource.FileStream;
allInputStreams[(int)folder.PackedStreamIndices[i]].InStreamIndex = packedStreamIndex;
}
List<Stream> inputCoders = new List<Stream>();
bool allCodersComplete = false;
while (!allCodersComplete)
{
allCodersComplete = true;
for (int i = 0; i < codersNeeded; i++)
{
Coder coder = folder.Coders[i];
// check is decoder already processed
if (coder.DecoderStream != null)
{
continue;
}
inputCoders.Clear();
for (int j = 0; j < (int)coder.NumInStreams; j++)
{
if (coder.InputStreamsSourceInfo[j].InStreamSource == InStreamSource.FileStream)
{
inputCoders.Add(_header.StreamsInfo.PackedStreams[coder.InputStreamsSourceInfo[j].InStreamIndex].PackedStream);
}
else if (coder.InputStreamsSourceInfo[j].InStreamSource == InStreamSource.CompStreamOutput)
{
if (folder.Coders[coder.InputStreamsSourceInfo[j].InStreamIndex].DecoderStream == null)
{
break;
}
inputCoders.Add(folder.Coders[coder.InputStreamsSourceInfo[j].InStreamIndex].DecoderStream);
}
else
{
// unknown input type so error
return ZipReturn.ZipDecodeError;
}
}
if (inputCoders.Count == (int)coder.NumInStreams)
{
// all inputs streams are available to make the decoder stream
switch (coder.DecoderType)
{
case DecompressType.Stored:
coder.DecoderStream = inputCoders[0];
break;
case DecompressType.Delta:
coder.DecoderStream = new Delta(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.LZMA:
coder.DecoderStream = new LzmaStream(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.LZMA2:
coder.DecoderStream = new LzmaStream(folder.Coders[i].Properties, inputCoders[0]);
break;
case DecompressType.PPMd:
coder.DecoderStream = new PpmdStream(new PpmdProperties(folder.Coders[i].Properties), inputCoders[0], false);
break;
case DecompressType.BZip2:
coder.DecoderStream = new CBZip2InputStream(inputCoders[0], false);
break;
case DecompressType.BCJ:
coder.DecoderStream = new BCJFilter(false, inputCoders[0]);
break;
case DecompressType.BCJ2:
coder.DecoderStream = new BCJ2Filter(inputCoders[0], inputCoders[1], inputCoders[2], inputCoders[3]);
break;
case DecompressType.ZSTD:
coder.DecoderStream = new ZstandardStream(inputCoders[0], CompressionMode.Decompress, true);
break;
default:
return ZipReturn.ZipDecodeError;
}
}
// if skipped a coder need to loop round again
if (coder.DecoderStream == null)
{
allCodersComplete = false;
}
}
}
// find the final output stream and return it.
int outputStream = -1;
for (int i = 0; i < codersNeeded; i++)
{
Coder coder = folder.Coders[i];
if (!coder.OutputUsedInternally)
{
outputStream = i;
}
}
stream = folder.Coders[outputStream].DecoderStream;
stream.Seek((long)_localFiles[index].StreamOffset, SeekOrigin.Current);
_stream = stream;
return ZipReturn.ZipGood;
}
catch (Exception e)
{
return ZipReturn.ZipErrorGettingDataStream;
}
}
private Stream CloneStream(Stream s)
{
switch (s)
{
case System.IO.FileStream _:
int errorCode = FileStream.OpenFileRead(ZipFilename, out Stream streamOut);
return errorCode != 0 ? null : streamOut;
case MemoryStream memStream:
long pos = memStream.Position;
memStream.Position = 0;
byte[] newStream = new byte[memStream.Length];
memStream.Read(newStream, 0, (int)memStream.Length);
MemoryStream ret = new MemoryStream(newStream, false);
memStream.Position = pos;
return ret;
}
return null;
}
public ZipReturn ZipFileCloseReadStream()
{
if (_streamIndex != -1)
{
Folder folder = _header.StreamsInfo.Folders[_streamIndex];
foreach (Coder c in folder.Coders)
{
Stream ds = c?.DecoderStream;
if (ds == null)
{
continue;
}
ds.Close();
ds.Dispose();
c.DecoderStream = null;
}
}
_streamIndex = -1;
if (_header?.StreamsInfo != null)
{
foreach (PackedStreamInfo psi in _header.StreamsInfo.PackedStreams)
{
if (psi?.PackedStream == null)
{
continue;
}
psi.PackedStream.Close();
psi.PackedStream.Dispose();
psi.PackedStream = null;
}
}
return ZipReturn.ZipGood;
}
}
}

View File

@@ -0,0 +1,157 @@
using System.IO;
using System.Text;
namespace Compress.SevenZip
{
public partial class SevenZ
{
// not finalized yet, so do not use
private void WriteRomVault7Zip(BinaryWriter bw, ulong headerPos, ulong headerLength, uint headerCRC)
{
const string sig = "RomVault7Z01";
byte[] RV7Zid = Util.Enc.GetBytes(sig);
// RomVault 7Zip torrent header
// 12 bytes : RomVault7Zip
// 4 bytes : HeaderCRC
// 8 bytes : HeaderPos
// 8 bytes : HeaderLength
bw.Write(RV7Zid);
bw.Write(headerCRC);
bw.Write(headerPos);
bw.Write(headerLength);
ZipStatus = ZipStatus.TrrntZip;
}
private bool IsRomVault7Z(long testBaseOffset, ulong testHeaderPos, ulong testHeaderLength, uint testHeaderCRC)
{
long length = _zipFs.Length;
if (length < 32)
{
return false;
}
_zipFs.Seek(_baseOffset + (long)testHeaderPos - 32, SeekOrigin.Begin);
const string sig = "RomVault7Z01";
byte[] rv7Zid = Util.Enc.GetBytes(sig);
byte[] header = new byte[12];
_zipFs.Read(header, 0, 12);
for (int i = 0; i < 12; i++)
{
if (header[i] != rv7Zid[i])
{
return false;
}
}
uint headerCRC;
ulong headerOffset; // is location of header in file
ulong headerSize;
using (BinaryReader br = new BinaryReader(_zipFs, Encoding.UTF8, true))
{
headerCRC = br.ReadUInt32();
headerOffset = br.ReadUInt64();
headerSize = br.ReadUInt64();
}
if (headerCRC != testHeaderCRC)
return false;
if (headerOffset != testHeaderPos + (ulong)testBaseOffset)
return false;
return headerSize == testHeaderLength;
}
private bool Istorrent7Z()
{
const int crcsz = 128;
const int t7ZsigSize = 16 + 1 + 9 + 4 + 4;
byte[] kSignature = { (byte)'7', (byte)'z', 0xBC, 0xAF, 0x27, 0x1C };
int kSignatureSize = kSignature.Length;
const string sig = "\xa9\x9f\xd1\x57\x08\xa9\xd7\xea\x29\x64\xb2\x36\x1b\x83\x52\x33\x01torrent7z_0.9beta";
byte[] t7Zid = Util.Enc.GetBytes(sig);
int t7ZidSize = t7Zid.Length;
const int tmpbufsize = 256 + t7ZsigSize + 8 + 4;
byte[] buffer = new byte[tmpbufsize];
// read fist 128 bytes, pad with zeros if less bytes
int bufferPos = 0;
_zipFs.Seek(0, SeekOrigin.Begin);
int ar = _zipFs.Read(buffer, bufferPos, crcsz);
if (ar < crcsz)
{
Util.memset(buffer, bufferPos + ar, 0, crcsz - ar);
}
bufferPos = crcsz;
long foffs = _zipFs.Length;
int endReadLength = crcsz + t7ZsigSize + 4;
foffs = foffs < endReadLength ? 0 : foffs - endReadLength;
_zipFs.Seek(foffs, SeekOrigin.Begin);
ar = _zipFs.Read(buffer, bufferPos, endReadLength);
if (ar < endReadLength)
{
if (ar >= t7ZsigSize + 4)
{
ar -= t7ZsigSize + 4;
}
if (ar < kSignatureSize)
{
ar = kSignatureSize;
}
Util.memset(buffer, bufferPos + ar, 0, crcsz - ar);
Util.memcpyr(buffer, crcsz * 2 + 8, buffer, bufferPos + ar, t7ZsigSize + 4);
}
else
{
Util.memcpyr(buffer, crcsz * 2 + 8, buffer, crcsz * 2, t7ZsigSize + 4);
}
foffs = _zipFs.Length;
foffs -= t7ZsigSize + 4;
//memcpy(buffer, crcsz * 2, &foffs, 8);
buffer[crcsz * 2 + 0] = (byte)((foffs >> 0) & 0xff);
buffer[crcsz * 2 + 1] = (byte)((foffs >> 8) & 0xff);
buffer[crcsz * 2 + 2] = (byte)((foffs >> 16) & 0xff);
buffer[crcsz * 2 + 3] = (byte)((foffs >> 24) & 0xff);
buffer[crcsz * 2 + 4] = 0;
buffer[crcsz * 2 + 5] = 0;
buffer[crcsz * 2 + 6] = 0;
buffer[crcsz * 2 + 7] = 0;
if (Util.memcmp(buffer, 0, kSignature, kSignatureSize))
{
t7Zid[16] = buffer[crcsz * 2 + 4 + 8 + 16];
if (Util.memcmp(buffer, crcsz * 2 + 4 + 8, t7Zid, t7ZidSize))
{
uint inCrc32 = (uint)(buffer[crcsz * 2 + 8 + 0] +
(buffer[crcsz * 2 + 8 + 1] << 8) +
(buffer[crcsz * 2 + 8 + 2] << 16) +
(buffer[crcsz * 2 + 8 + 3] << 24));
buffer[crcsz * 2 + 8 + 0] = 0xff;
buffer[crcsz * 2 + 8 + 1] = 0xff;
buffer[crcsz * 2 + 8 + 2] = 0xff;
buffer[crcsz * 2 + 8 + 3] = 0xff;
uint calcCrc32 = Utils.CRC.CalculateDigest(buffer, 0, crcsz * 2 + 8 + t7ZsigSize + 4);
if (inCrc32 == calcCrc32)
{
return true;
}
}
}
return false;
}
}
}

View File

@@ -0,0 +1,176 @@
using System.IO;
using System.Text;
using Compress.SevenZip.Compress.LZMA;
using Compress.SevenZip.Structure;
using Compress.Utils;
using FileInfo = RVIO.FileInfo;
using FileStream = RVIO.FileStream;
namespace Compress.SevenZip
{
public partial class SevenZ
{
private Stream _lzmaStream;
private ulong _packStreamStart;
private ulong _packStreamSize;
private ulong _unpackedStreamSize;
private byte[] _codeMSbytes;
public ZipReturn ZipFileCreate(string newFilename)
{
return ZipFileCreate(newFilename, true);
}
public ZipReturn ZipFileCreateFromUncompressedSize(string newFilename, ulong unCompressedSize)
{
return ZipFileCreate(newFilename, true, GetDictionarySizeFromUncompressedSize(unCompressedSize));
}
public ZipReturn ZipFileCreate(string newFilename, bool compressOutput, int dictionarySize = 1 << 24, int numFastBytes = 64)
{
if (ZipOpen != ZipOpenType.Closed)
{
return ZipReturn.ZipFileAlreadyOpen;
}
DirUtil.CreateDirForFile(newFilename);
_zipFileInfo = new FileInfo(newFilename);
int errorCode = FileStream.OpenFileWrite(newFilename, out _zipFs);
if (errorCode != 0)
{
ZipFileClose();
return ZipReturn.ZipErrorOpeningFile;
}
ZipOpen = ZipOpenType.OpenWrite;
_signatureHeader = new SignatureHeader();
_header = new Header();
using (BinaryWriter bw = new BinaryWriter(_zipFs, Encoding.UTF8, true))
{
_signatureHeader.Write(bw);
}
_baseOffset = _zipFs.Position;
_compressed = compressOutput;
_unpackedStreamSize = 0;
if (_compressed)
{
LzmaEncoderProperties ep = new LzmaEncoderProperties(true, dictionarySize, numFastBytes);
LzmaStream lzs = new LzmaStream(ep, false, _zipFs);
_codeMSbytes = lzs.Properties;
_lzmaStream = lzs;
/*
ZstandardStream zss = new ZstandardStream(_zipFs, 22, true);
_codeMSbytes = new byte[] { 1, 4, 18, 0, 0 };
_lzmaStream = zss;
*/
_packStreamStart = (ulong)_zipFs.Position;
}
return ZipReturn.ZipGood;
}
public void ZipFileAddDirectory(string filename)
{
string fName = filename;
if (fName.Substring(fName.Length - 1, 1) == @"/")
fName = fName.Substring(0, fName.Length - 1);
LocalFile lf = new LocalFile
{
FileName = fName,
UncompressedSize = 0,
IsDirectory = true,
StreamOffset = 0
};
_localFiles.Add(lf);
}
public void ZipFileAddZeroLengthFile()
{
// do nothing here for 7zip
}
public ZipReturn ZipFileOpenWriteStream(bool raw, bool trrntzip, string filename, ulong uncompressedSize, ushort compressionMethod, out Stream stream)
{
return ZipFileOpenWriteStream(filename, uncompressedSize, out stream);
}
private ZipReturn ZipFileOpenWriteStream(string filename, ulong uncompressedSize, out Stream stream)
{
LocalFile lf = new LocalFile
{
FileName = filename,
UncompressedSize = uncompressedSize,
StreamOffset = (ulong)(_zipFs.Position - _signatureHeader.BaseOffset)
};
if (uncompressedSize == 0 && filename.Substring(filename.Length - 1, 1) == "/")
{
lf.FileName = filename.Substring(0, filename.Length - 1);
lf.IsDirectory = true;
}
_unpackedStreamSize += uncompressedSize;
_localFiles.Add(lf);
stream = _compressed ? _lzmaStream : _zipFs;
return ZipReturn.ZipGood;
}
public ZipReturn ZipFileCloseWriteStream(byte[] crc32)
{
_localFiles[_localFiles.Count - 1].CRC = new[] { crc32[3], crc32[2], crc32[1], crc32[0] };
return ZipReturn.ZipGood;
}
private static readonly int[] DictionarySizes =
{
0x10000,
0x18000,
0x20000,
0x30000,
0x40000,
0x60000,
0x80000,
0xc0000,
0x100000,
0x180000,
0x200000,
0x300000,
0x400000,
0x600000,
0x800000,
0xc00000,
0x1000000,
0x1800000,
0x2000000,
0x3000000,
0x4000000,
0x6000000
};
private static int GetDictionarySizeFromUncompressedSize(ulong unCompressedSize)
{
foreach (int v in DictionarySizes)
{
if ((ulong)v >= unCompressedSize)
return v;
}
return DictionarySizes[DictionarySizes.Length - 1];
}
}
}

View File

@@ -0,0 +1,293 @@
using System.IO;
using System.Text;
using Compress.SevenZip.Compress.LZMA;
using Compress.SevenZip.Structure;
using Compress.Utils;
using Zstandard.Net;
namespace Compress.SevenZip
{
public partial class SevenZ
{
private void Create7ZStructure()
{
int fileCount = _localFiles.Count;
//FileInfo
_header.FileInfo = new Structure.FileInfo
{
Names = new string[fileCount]
};
ulong emptyStreamCount = 0;
ulong emptyFileCount = 0;
for (int i = 0; i < fileCount; i++)
{
_header.FileInfo.Names[i] = _localFiles[i].FileName;
if (_localFiles[i].UncompressedSize != 0)
{
continue;
}
if (!_localFiles[i].IsDirectory)
{
emptyFileCount += 1;
}
emptyStreamCount += 1;
}
ulong outFileCount = (ulong)_localFiles.Count - emptyStreamCount;
_header.FileInfo.EmptyStreamFlags = null;
_header.FileInfo.EmptyFileFlags = null;
_header.FileInfo.Attributes = null;
if (emptyStreamCount > 0)
{
if (emptyStreamCount != emptyFileCount) //then we found directories and need to set the attributes
{
_header.FileInfo.Attributes = new uint[fileCount];
}
if (emptyFileCount > 0)
{
_header.FileInfo.EmptyFileFlags = new bool[emptyStreamCount];
}
emptyStreamCount = 0;
_header.FileInfo.EmptyStreamFlags = new bool[fileCount];
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize != 0)
{
continue;
}
if (_localFiles[i].IsDirectory)
{
if (_header.FileInfo.Attributes != null)
_header.FileInfo.Attributes[i] = 0x10; // set attributes to directory
}
else
{
if (_header.FileInfo.EmptyFileFlags != null)
_header.FileInfo.EmptyFileFlags[emptyStreamCount] = true; // set empty file flag
}
_header.FileInfo.EmptyStreamFlags[i] = true;
emptyStreamCount += 1;
}
}
//StreamsInfo
_header.StreamsInfo = new StreamsInfo { PackPosition = 0 };
//StreamsInfo.PackedStreamsInfo
if (_compressed)
{
_header.StreamsInfo.PackedStreams = new PackedStreamInfo[1];
_header.StreamsInfo.PackedStreams[0] = new PackedStreamInfo { PackedSize = _packStreamSize };
}
else
{
_header.StreamsInfo.PackedStreams = new PackedStreamInfo[outFileCount];
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
_header.StreamsInfo.PackedStreams[fileIndex++] = new PackedStreamInfo { PackedSize = _localFiles[i].UncompressedSize };
}
}
//StreamsInfo.PackedStreamsInfo, no CRC or StreamPosition required
if (_compressed)
{
//StreamsInfo.Folders
_header.StreamsInfo.Folders = new Folder[1];
//StreamsInfo.Folders.Coder
// flags 0x23
Folder folder = new Folder
{
BindPairs = null,
Coders = new[] {
new Coder {
Method = new byte[] { 3, 1, 1 },
NumInStreams = 1,
NumOutStreams = 1,
Properties = _codeMSbytes
}
},
PackedStreamIndices = new[] { (ulong)0 },
UnpackedStreamSizes = new[] { _unpackedStreamSize },
UnpackedStreamInfo = new UnpackedStreamInfo[outFileCount],
UnpackCRC = null
};
switch (_lzmaStream)
{
case LzmaStream _:
folder.Coders[0].Method = new byte[] { 3, 1, 1 };
break;
case ZstandardStream _:
folder.Coders[0].Method = new byte[] { 4, 247, 17, 1 };
break;
}
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
UnpackedStreamInfo unpackedStreamInfo = new UnpackedStreamInfo
{
UnpackedSize = _localFiles[i].UncompressedSize,
Crc = Util.bytestouint(_localFiles[i].CRC)
};
folder.UnpackedStreamInfo[fileIndex++] = unpackedStreamInfo;
}
_header.StreamsInfo.Folders[0] = folder;
}
else
{
_header.StreamsInfo.Folders = new Folder[outFileCount];
int fileIndex = 0;
for (int i = 0; i < fileCount; i++)
{
if (_localFiles[i].UncompressedSize == 0)
{
continue;
}
Folder folder = new Folder
{
BindPairs = null,
Coders = new[] {
new Coder {
Method = new byte[] {0},
NumInStreams = 1,
NumOutStreams = 1,
Properties = null
}
},
PackedStreamIndices = new[] { (ulong)i },
UnpackedStreamSizes = new[] { _localFiles[i].UncompressedSize },
UnpackCRC = null,
UnpackedStreamInfo = new[] {
new UnpackedStreamInfo {
UnpackedSize = _localFiles[i].UncompressedSize,
Crc = Util.bytestouint(_localFiles[i].CRC)
}
}
};
_header.StreamsInfo.Folders[fileIndex++] = folder;
}
}
}
private void CloseWriting7Zip()
{
if (_compressed)
{
_lzmaStream.Close();
}
_packStreamSize = (ulong)_zipFs.Position - _packStreamStart;
Create7ZStructure();
byte[] newHeaderByte;
using (Stream headerMem = new MemoryStream())
{
using (BinaryWriter headerBw = new BinaryWriter(headerMem, Encoding.UTF8, true))
{
_header.WriteHeader(headerBw);
newHeaderByte = new byte[headerMem.Length];
headerMem.Position = 0;
headerMem.Read(newHeaderByte, 0, newHeaderByte.Length);
}
}
uint mainHeaderCRC = CRC.CalculateDigest(newHeaderByte, 0, (uint)newHeaderByte.Length);
#region Header Compression
long packedHeaderPos = _zipFs.Position;
LzmaEncoderProperties ep = new LzmaEncoderProperties(true, GetDictionarySizeFromUncompressedSize((ulong)newHeaderByte.Length), 64);
LzmaStream lzs = new LzmaStream(ep, false, _zipFs);
byte[] lzmaStreamProperties = lzs.Properties;
lzs.Write(newHeaderByte, 0, newHeaderByte.Length);
lzs.Close();
StreamsInfo streamsInfo = new StreamsInfo
{
PackPosition = (ulong)(packedHeaderPos - _baseOffset),
Folders = new[] {
new Folder {
BindPairs = new BindPair[0],
Coders = new [] {
new Coder {
Method = new byte[] { 3, 1, 1 },
NumInStreams = 1,
NumOutStreams = 1,
Properties = lzmaStreamProperties
}
},
UnpackedStreamSizes = new[] {(ulong) newHeaderByte.Length},
UnpackCRC = mainHeaderCRC
}
},
PackedStreams = new[] {
new PackedStreamInfo
{
PackedSize = (ulong)(_zipFs.Position - packedHeaderPos),
StreamPosition = 0
}
}
};
using (Stream headerMem = new MemoryStream())
{
using (BinaryWriter bw = new BinaryWriter(headerMem, Encoding.UTF8, true))
{
bw.Write((byte)HeaderProperty.kEncodedHeader);
streamsInfo.WriteHeader(bw);
newHeaderByte = new byte[headerMem.Length];
headerMem.Position = 0;
headerMem.Read(newHeaderByte, 0, newHeaderByte.Length);
}
}
mainHeaderCRC = CRC.CalculateDigest(newHeaderByte, 0, (uint)newHeaderByte.Length);
#endregion
using (BinaryWriter bw = new BinaryWriter(_zipFs, Encoding.UTF8, true))
{
ulong headerPosition = (ulong)_zipFs.Position + 32; //tzip header is 32 bytes
WriteRomVault7Zip(bw, headerPosition, (ulong)newHeaderByte.Length, mainHeaderCRC);
_zipFs.Write(newHeaderByte, 0, newHeaderByte.Length);
_signatureHeader.WriteFinal(bw, headerPosition, (ulong)newHeaderByte.Length, mainHeaderCRC);
}
_zipFs.Flush();
_zipFs.Close();
_zipFs.Dispose();
}
}
}

View File

@@ -29,7 +29,8 @@ namespace Compress.SevenZip.Structure
BCJ2, BCJ2,
PPMd, PPMd,
BZip2, BZip2,
LZMA2 LZMA2,
ZSTD
} }
@@ -73,40 +74,42 @@ namespace Compress.SevenZip.Structure
throw new NotSupportedException("External flag"); throw new NotSupportedException("External flag");
} }
if ((Method.Length == 1) && (Method[0] == 0)) if (Method.Length == 1 && Method[0] == 0)
{ {
DecoderType = DecompressType.Stored; DecoderType = DecompressType.Stored;
} }
else if ((Method.Length == 1) && (Method[0] == 3)) else if (Method.Length == 1 && Method[0] == 3)
{ {
DecoderType = DecompressType.Delta; DecoderType = DecompressType.Delta;
} }
else if ((Method.Length == 3) && (Method[0] == 3) && (Method[1] == 1) && (Method[2] == 1)) else if (Method.Length == 3 && Method[0] == 3 && Method[1] == 1 && Method[2] == 1)
{ {
DecoderType = DecompressType.LZMA; DecoderType = DecompressType.LZMA;
} }
else if ((Method.Length == 4) && (Method[0] == 3) && (Method[1] == 3) && (Method[2] == 1) && else if (Method.Length == 4 && Method[0] == 3 && Method[1] == 3 && Method[2] == 1 && Method[3] == 3)
(Method[3] == 3))
{ {
DecoderType = DecompressType.BCJ; DecoderType = DecompressType.BCJ;
} }
else if ((Method.Length == 4) && (Method[0] == 3) && (Method[1] == 3) && (Method[2] == 1) && else if (Method.Length == 4 && Method[0] == 3 && Method[1] == 3 && Method[2] == 1 && Method[3] == 27)
(Method[3] == 27))
{ {
DecoderType = DecompressType.BCJ2; DecoderType = DecompressType.BCJ2;
} }
else if ((Method.Length == 3) && (Method[0] == 3) && (Method[1] == 4) && (Method[2] == 1)) else if (Method.Length == 3 && Method[0] == 3 && Method[1] == 4 && Method[2] == 1)
{ {
DecoderType = DecompressType.PPMd; DecoderType = DecompressType.PPMd;
} }
else if ((Method.Length == 3) && (Method[0] == 4) && (Method[1] == 2) && (Method[2] == 2)) else if (Method.Length == 3 && Method[0] == 4 && Method[1] == 2 && Method[2] == 2)
{ {
DecoderType = DecompressType.BZip2; DecoderType = DecompressType.BZip2;
} }
else if ((Method.Length == 1) && (Method[0] == 33)) else if (Method.Length == 1 && Method[0] == 33)
{ {
DecoderType = DecompressType.LZMA2; DecoderType = DecompressType.LZMA2;
} }
else if (SevenZ.supportZstd && Method.Length == 4 && Method[0] == 4 && Method[1] == 247 && Method[2] == 17 && Method[3] == 1)
{
DecoderType = DecompressType.ZSTD;
}
InputStreamsSourceInfo = new InStreamSourceInfo[NumInStreams]; InputStreamsSourceInfo = new InStreamSourceInfo[NumInStreams];
for (uint i = 0; i < NumInStreams; i++) for (uint i = 0; i < NumInStreams; i++)

View File

@@ -338,10 +338,18 @@ namespace Compress.SevenZip.Structure
Folders[i].WriteUnpackedStreamSize(bw); Folders[i].WriteUnpackedStreamSize(bw);
} }
if (Folders[0].UnpackCRC != null) bool hasCRC = false;
uint?[] CRCs = new uint?[numFolders];
for (ulong i = 0; i < numFolders; i++)
{
CRCs[i] = Folders[i].UnpackCRC;
hasCRC |= (CRCs[i] != null);
}
if (hasCRC)
{ {
bw.Write((byte)HeaderProperty.kCRC); bw.Write((byte)HeaderProperty.kCRC);
throw new NotImplementedException(); Util.WritePackedCRCs(bw, CRCs);
} }
bw.Write((byte)HeaderProperty.kEnd); bw.Write((byte)HeaderProperty.kEnd);

View File

@@ -38,7 +38,7 @@ namespace Compress.SevenZip.Structure
} }
private void Write(BinaryWriter bw) public void WriteHeader(BinaryWriter bw)
{ {
bw.Write((byte)HeaderProperty.kHeader); bw.Write((byte)HeaderProperty.kHeader);
StreamsInfo.Write(bw); StreamsInfo.Write(bw);
@@ -46,11 +46,6 @@ namespace Compress.SevenZip.Structure
bw.Write((byte)HeaderProperty.kEnd); bw.Write((byte)HeaderProperty.kEnd);
} }
public void WriteHeader(BinaryWriter bw)
{
Write(bw);
}
public static ZipReturn ReadHeaderOrPackedHeader(Stream stream, long baseOffset, out Header header) public static ZipReturn ReadHeaderOrPackedHeader(Stream stream, long baseOffset, out Header header)
{ {
header = null; header = null;

View File

@@ -49,6 +49,13 @@ namespace Compress.SevenZip.Structure
bw.Write((byte)HeaderProperty.kEnd); bw.Write((byte)HeaderProperty.kEnd);
} }
public void WriteHeader(BinaryWriter bw)
{
PackedStreamInfo.Write(bw, PackPosition, PackedStreams);
Folder.WriteUnPackInfo(bw, Folders);
bw.Write((byte)HeaderProperty.kEnd);
}
public void Report(ref StringBuilder sb) public void Report(ref StringBuilder sb)
{ {

View File

@@ -183,6 +183,23 @@ namespace Compress.SevenZip
} }
} }
public static void WritePackedCRCs(BinaryWriter bw, uint?[] digests)
{
bool[] digestsDefined = new bool[digests.Length];
for (int i = 0; i < digests.Length; i++)
{
digestsDefined[i] = digests[i] != null;
}
WriteBoolFlagsDefaultTrue(bw, digestsDefined);
for (int i = 0; i < digests.Length; i++)
{
if (digestsDefined[i])
{
bw.Write((uint)digests[i]);
}
}
}
private static bool[] ReadBoolFlagsDefaultTrue(BinaryReader br, ulong numItems) private static bool[] ReadBoolFlagsDefaultTrue(BinaryReader br, ulong numItems)
{ {
byte allAreDefined = br.ReadByte(); byte allAreDefined = br.ReadByte();
@@ -198,6 +215,22 @@ namespace Compress.SevenZip
return flags; return flags;
} }
private static void WriteBoolFlagsDefaultTrue(BinaryWriter bw, bool[] bArray)
{
bool allTrue = true;
foreach (bool b in bArray)
{
allTrue &= b;
}
if (allTrue)
{
bw.Write((byte)1);
return;
}
WriteBoolFlags(bw, bArray);
}
public static bool[] ReadBoolFlags(BinaryReader br, ulong numItems) public static bool[] ReadBoolFlags(BinaryReader br, ulong numItems)
{ {
byte b = 0; byte b = 0;

View File

@@ -29,6 +29,7 @@ namespace Compress
ZipErrorTimeStamp, ZipErrorTimeStamp,
ZipErrorRollBackFile, ZipErrorRollBackFile,
ZipTryingToAccessADirectory, ZipTryingToAccessADirectory,
ZipErrorWritingToOutputStream,
ZipUntested ZipUntested
} }

View File

@@ -382,7 +382,8 @@ namespace Compress.ZipFile.ZLib
// The distribution counts are first used to generate the code values // The distribution counts are first used to generate the code values
// without bit reversal. // without bit reversal.
for (bits = 1; bits <= InternalConstants.MAX_BITS; bits++) for (bits = 1; bits <= InternalConstants.MAX_BITS; bits++)
unchecked { unchecked
{
next_code[bits] = code = (short)((code + bl_count[bits - 1]) << 1); next_code[bits] = code = (short)((code + bl_count[bits - 1]) << 1);
} }

View File

@@ -123,4 +123,3 @@ namespace Compress.ZipFile.ZLib
} }
} }

View File

@@ -138,10 +138,6 @@ namespace Compress.ZipFile
bool lTrrntzip = true; bool lTrrntzip = true;
_centerDirStart = (ulong)_zipFs.Position; _centerDirStart = (ulong)_zipFs.Position;
if (_centerDirStart >= 0xffffffff)
{
_zip64 = true;
}
using (CrcCalculatorStream crcCs = new CrcCalculatorStream(_zipFs, true)) using (CrcCalculatorStream crcCs = new CrcCalculatorStream(_zipFs, true))
{ {
@@ -160,6 +156,9 @@ namespace Compress.ZipFile
_fileComment = lTrrntzip ? GetBytes("TORRENTZIPPED-" + crcCs.Crc.ToString("X8")) : new byte[0]; _fileComment = lTrrntzip ? GetBytes("TORRENTZIPPED-" + crcCs.Crc.ToString("X8")) : new byte[0];
ZipStatus = lTrrntzip ? ZipStatus.TrrntZip : ZipStatus.None; ZipStatus = lTrrntzip ? ZipStatus.TrrntZip : ZipStatus.None;
} }
_zip64 |= _centerDirStart >= 0xffffffff;
_zip64 |= _centerDirSize >= 0xffffffff;
_zip64 |= _localFiles.Count >= 0xffff;
if (_zip64) if (_zip64)
{ {
@@ -218,7 +217,6 @@ namespace Compress.ZipFile
return ZipReturn.ZipGood; return ZipReturn.ZipGood;
} }
// TODO: Figure out how to re-add the file time functionality to this
public ZipReturn ZipFileOpenWriteStream(bool raw, bool trrntzip, string filename, ulong uncompressedSize, ushort compressionMethod, out Stream stream) public ZipReturn ZipFileOpenWriteStream(bool raw, bool trrntzip, string filename, ulong uncompressedSize, ushort compressionMethod, out Stream stream)
{ {
stream = null; stream = null;
@@ -271,9 +269,9 @@ namespace Compress.ZipFile
return ZipReturn.ZipGood; return ZipReturn.ZipGood;
} }
public void ZipFileAddDirectory() public void ZipFileAddZeroLengthFile()
{ {
LocalFile.LocalFileAddDirectory(_zipFs); LocalFile.LocalFileAddZeroLengthFile(_zipFs);
} }
/* /*
@@ -528,7 +526,7 @@ namespace Compress.ZipFile
// TODO: Figure out the timestamp instead of using ticks like we were before
public ZipReturn ZipFileOpen(string newFilename, long timestamp, bool readHeaders) public ZipReturn ZipFileOpen(string newFilename, long timestamp, bool readHeaders)
{ {
ZipFileClose(); ZipFileClose();
@@ -1799,9 +1797,13 @@ namespace Compress.ZipFile
if (_compressedSize == 0 && UncompressedSize == 0) if (_compressedSize == 0 && UncompressedSize == 0)
{ {
LocalFileAddDirectory(zipFs); LocalFileAddZeroLengthFile(zipFs);
_compressedSize = (ulong)zipFs.Position - _dataLocation; _compressedSize = (ulong)zipFs.Position - _dataLocation;
} }
else if (_compressedSize == 0 && UncompressedSize != 0)
{
return ZipReturn.ZipErrorWritingToOutputStream;
}
CRC = crc32; CRC = crc32;
WriteCompressedSize(zipFs); WriteCompressedSize(zipFs);
@@ -1892,7 +1894,7 @@ namespace Compress.ZipFile
zipFs.Seek(posNow, SeekOrigin.Begin); zipFs.Seek(posNow, SeekOrigin.Begin);
} }
public static void LocalFileAddDirectory(Stream zipFs) public static void LocalFileAddZeroLengthFile(Stream zipFs)
{ {
zipFs.WriteByte(03); zipFs.WriteByte(03);
zipFs.WriteByte(00); zipFs.WriteByte(00);

View File

@@ -326,7 +326,7 @@ namespace Compress.gZip
public long TimeStamp => _zipFileInfo?.LastWriteTime ?? 0; public long TimeStamp => _zipFileInfo?.LastWriteTime ?? 0;
public void ZipFileAddDirectory() public void ZipFileAddZeroLengthFile()
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }

View File

@@ -443,6 +443,8 @@ namespace RVIO
{ {
case 5: return "ERROR_ACCESS_DENIED: Access is denied."; case 5: return "ERROR_ACCESS_DENIED: Access is denied.";
case 32: return "ERROR_FILE_IN_USE: The file is in use by another process."; case 32: return "ERROR_FILE_IN_USE: The file is in use by another process.";
case 39: return "ERROR_HANDLE_DISK_FULL: The disk is full.";
case 112: return "ERROR_DISK_FULL: There is not enough space on the disk.";
case 123: return "ERROR_INVALID_NAME: The filename, directory name, or volume label syntax is incorrect."; case 123: return "ERROR_INVALID_NAME: The filename, directory name, or volume label syntax is incorrect.";
case 183: return "ERROR_ALREADY_EXISTS: Cannot create a file when that file already exists."; case 183: return "ERROR_ALREADY_EXISTS: Cannot create a file when that file already exists.";
} }

View File

@@ -18,18 +18,16 @@ using MemoryStream = System.IO.MemoryStream;
using SeekOrigin = System.IO.SeekOrigin; using SeekOrigin = System.IO.SeekOrigin;
using Stream = System.IO.Stream; using Stream = System.IO.Stream;
#endif #endif
using Compress;
using Compress.SevenZip;
using Compress.ZipFile; using Compress.ZipFile;
using SevenZip; // TODO: Remove this when 7zip write is implemented in SharpCompress using NaturalSort;
using SharpCompress.Archives;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Readers;
namespace SabreTools.Library.FileTypes namespace SabreTools.Library.FileTypes
{ {
/// <summary> /// <summary>
/// Represents a Torrent7zip archive for reading and writing /// Represents a Torrent7zip archive for reading and writing
/// </summary> /// </summary>
/// TODO: Torrent 7-zip: https://sourceforge.net/p/t7z/code/HEAD/tree/
public class SevenZipArchive : BaseArchive public class SevenZipArchive : BaseArchive
{ {
#region Constructors #region Constructors
@@ -74,13 +72,62 @@ namespace SabreTools.Library.FileTypes
Directory.CreateDirectory(outDir); Directory.CreateDirectory(outDir);
// Extract all files to the temp directory // Extract all files to the temp directory
SharpCompress.Archives.SevenZip.SevenZipArchive sza = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(Utilities.TryOpenRead(this.Filename)); SevenZ zf = new SevenZ();
foreach (SevenZipArchiveEntry entry in sza.Entries) ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{ {
entry.WriteToDirectory(outDir, new SharpCompress.Common.ExtractionOptions { PreserveFileTime = true, ExtractFullPath = true, Overwrite = true }); throw new Exception(ZipFile.ZipErrorMessageText(zr));
} }
for (int i = 0; i < zf.LocalFilesCount() && zr == ZipReturn.ZipGood; i++)
{
// Open the read stream
zr = zf.ZipFileOpenReadStream(i, out Stream readStream, out ulong streamsize);
// Create the rest of the path, if needed
if (!String.IsNullOrWhiteSpace(Path.GetDirectoryName(zf.Filename(i))))
{
Directory.CreateDirectory(Path.Combine(outDir, Path.GetDirectoryName(zf.Filename(i))));
}
// If the entry ends with a directory separator, continue to the next item, if any
if (zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
zf.ZipFileCloseReadStream();
continue;
}
FileStream writeStream = Utilities.TryCreate(Path.Combine(outDir, zf.Filename(i)));
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
// Otherwise, we do the normal loop
else
{
int realBufferSize = (streamsize < _bufferSize ? (int)streamsize : _bufferSize);
byte[] ibuffer = new byte[realBufferSize];
int ilen;
while ((ilen = readStream.Read(ibuffer, 0, realBufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
}
zr = zf.ZipFileCloseReadStream();
writeStream.Dispose();
}
zf.ZipFileClose();
encounteredErrors = false; encounteredErrors = false;
sza.Dispose();
} }
catch (EndOfStreamException) catch (EndOfStreamException)
{ {
@@ -158,18 +205,52 @@ namespace SabreTools.Library.FileTypes
try try
{ {
SharpCompress.Archives.SevenZip.SevenZipArchive sza = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(this.Filename, new ReaderOptions { LeaveStreamOpen = false, }); SevenZ zf = new SevenZ();
foreach (SevenZipArchiveEntry entry in sza.Entries) ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{ {
if (entry != null && !entry.IsDirectory && entry.Key.Contains(entryName)) throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.LocalFilesCount() && zr == ZipReturn.ZipGood; i++)
{ {
// Write the file out if (zf.Filename(i).Contains(entryName))
realEntry = entry.Key; {
entry.WriteTo(ms); // Open the read stream
break; realEntry = zf.Filename(i);
zr = zf.ZipFileOpenReadStream(i, out Stream readStream, out ulong streamsize);
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
// Otherwise, we do the normal loop
else
{
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while (streamsize > _bufferSize)
{
ilen = readStream.Read(ibuffer, 0, _bufferSize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
streamsize -= _bufferSize;
}
ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
zr = zf.ZipFileCloseReadStream();
} }
} }
sza.Dispose();
zf.ZipFileClose();
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -199,18 +280,47 @@ namespace SabreTools.Library.FileTypes
try try
{ {
SharpCompress.Archives.SevenZip.SevenZipArchive sza = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(Utilities.TryOpenRead(this.Filename)); SevenZ zf = new SevenZ();
foreach (SevenZipArchiveEntry entry in sza.Entries.Where(e => e != null && !e.IsDirectory)) ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{ {
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.LocalFilesCount(); i++)
{
// If the entry is a directory (or looks like a directory), we don't want to open it
if (zf.IsDirectory(i)
|| zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
continue;
}
// Open the read stream
zr = zf.ZipFileOpenReadStream(i, out Stream readStream, out ulong streamsize);
// If we get a read error, log it and continue
if (zr != ZipReturn.ZipGood)
{
Globals.Logger.Warning("An error occurred while reading archive {0}: Zip Error - {1}", this.Filename, zr);
zr = zf.ZipFileCloseReadStream();
continue;
}
// If secure hashes are disabled, do a quickscan // If secure hashes are disabled, do a quickscan
if (omitFromScan == Hash.SecureHashes) if (omitFromScan == Hash.SecureHashes)
{ {
string newname = zf.Filename(i);
long newsize = (long)zf.UncompressedSize(i);
byte[] newcrc = zf.CRC32(i);
found.Add(new BaseFile found.Add(new BaseFile
{ {
Filename = entry.Key, Filename = newname,
Size = entry.Size, Size = newsize,
CRC = BitConverter.GetBytes(entry.Crc), CRC = newcrc,
Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null),
Parent = gamename, Parent = gamename,
}); });
@@ -218,18 +328,16 @@ namespace SabreTools.Library.FileTypes
// Otherwise, use the stream directly // Otherwise, use the stream directly
else else
{ {
Stream entryStream = entry.OpenEntryStream(); BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true);
BaseFile sevenZipEntryRom = Utilities.GetStreamInfo(entryStream, entry.Size, omitFromScan: omitFromScan); zipEntryRom.Filename = zf.Filename(i);
sevenZipEntryRom.Filename = entry.Key; zipEntryRom.Parent = gamename;
sevenZipEntryRom.Parent = gamename; found.Add(zipEntryRom);
sevenZipEntryRom.Date = (date && entry.LastModifiedTime != null ? entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss") : null);
found.Add(sevenZipEntryRom);
entryStream.Dispose();
} }
} }
// Dispose of the archive // Dispose of the archive
sza.Dispose(); zr = zf.ZipFileCloseReadStream();
zf.ZipFileClose();
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -251,24 +359,36 @@ namespace SabreTools.Library.FileTypes
try try
{ {
SharpCompress.Archives.SevenZip.SevenZipArchive sza = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(this.Filename, new ReaderOptions { LeaveStreamOpen = false }); SevenZ zf = new SevenZ();
List<SevenZipArchiveEntry> sevenZipEntries = sza.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
string lastSevenZipEntry = null; if (zr != ZipReturn.ZipGood)
foreach (SevenZipArchiveEntry entry in sevenZipEntries)
{ {
if (entry != null) throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
List<(string, bool)> zipEntries = new List<(string, bool)>();
for (int i = 0; i < zf.LocalFilesCount(); i++)
{
zipEntries.Add((zf.Filename(i), zf.IsDirectory(i)));
}
zipEntries = zipEntries.OrderBy(p => p.Item1, new NaturalReversedComparer()).ToList();
string lastZipEntry = null;
foreach ((string, bool) entry in zipEntries)
{ {
// If the current is a superset of last, we skip it // If the current is a superset of last, we skip it
if (lastSevenZipEntry != null && lastSevenZipEntry.StartsWith(entry.Key)) if (lastZipEntry != null && lastZipEntry.StartsWith(entry.Item1))
{ {
// No-op // No-op
} }
// If the entry is a directory, we add it // If the entry is a directory, we add it
else if (entry.IsDirectory) else
{ {
empties.Add(entry.Key); if (entry.Item2)
lastSevenZipEntry = entry.Key; {
empties.Add(entry.Item1);
} }
lastZipEntry = entry.Item1;
} }
} }
} }
@@ -283,43 +403,16 @@ namespace SabreTools.Library.FileTypes
/// <summary> /// <summary>
/// Check whether the input file is a standardized format /// Check whether the input file is a standardized format
/// </summary> /// </summary>
/// TODO: Finish reading T7z information
public override bool IsTorrent() public override bool IsTorrent()
{ {
bool ist7z = false; SevenZ zf = new SevenZ();
ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (File.Exists(this.Filename)) if (zr != ZipReturn.ZipGood)
{ {
try throw new Exception(ZipFile.ZipErrorMessageText(zr));
{
Stream fread = Utilities.TryOpenRead(this.Filename);
uint ar, offs = 0;
fread.Seek(0, SeekOrigin.Begin);
byte[] buffer = new byte[128];
ar = (uint)fread.Read(buffer, 0, 4 + Constants.Torrent7ZipSignature.Length + 4);
if (ar < (4 + Constants.Torrent7ZipSignature.Length + 4))
{
if (ar >= Constants.Torrent7ZipSignature.Length + 4)
{
ar -= (uint)(Constants.Torrent7ZipSignature.Length + 4);
}
if (ar <= Constants.Torrent7ZipHeader.Length)
{
ar = (uint)Constants.Torrent7ZipHeader.Length;
}
// memset(buffer+offs+ar,0,crcsz-ar)
} }
fread.Dispose(); return zf.ZipStatus == ZipStatus.Trrnt7Zip;
}
catch
{
Globals.Logger.Warning("File '{0}' could not be opened", this.Filename);
ist7z = false;
}
}
return ist7z;
} }
#endregion #endregion
@@ -371,79 +464,89 @@ namespace SabreTools.Library.FileTypes
inputStream.Seek(0, SeekOrigin.Begin); inputStream.Seek(0, SeekOrigin.Begin);
// Get the output archive name from the first rebuild rom // Get the output archive name from the first rebuild rom
string archiveFileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".7z") ? "" : ".7z")); string archiveFileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(rom.MachineName) + (rom.MachineName.EndsWith(".zip") ? "" : ".zip"));
// Set internal variables // Set internal variables
SevenZipBase.SetLibraryPath("7za.dll"); Stream writeStream = null;
SevenZipExtractor oldZipFile = null; SevenZ oldZipFile = new SevenZ();
SevenZipCompressor zipFile; SevenZ zipFile = new SevenZ();
ZipReturn zipReturn = ZipReturn.ZipGood;
try try
{ {
// If the full output path doesn't exist, create it // If the full output path doesn't exist, create it
if (!Directory.Exists(Path.GetDirectoryName(tempFile))) if (!Directory.Exists(Path.GetDirectoryName(archiveFileName)))
{ {
Directory.CreateDirectory(Path.GetDirectoryName(tempFile)); Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName));
} }
// If the archive doesn't exist, create it and put the single file // If the archive doesn't exist, create it and put the single file
if (!File.Exists(archiveFileName)) if (!File.Exists(archiveFileName))
{ {
zipFile = new SevenZipCompressor() inputStream.Seek(0, SeekOrigin.Begin);
{ zipReturn = zipFile.ZipFileCreate(tempFile);
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionLevel = CompressionLevel.Normal,
};
// Create the temp directory // Open the input file for reading
string tempPath = Path.Combine(outDir, Guid.NewGuid().ToString()); ulong istreamSize = (ulong)(inputStream.Length);
if (!Directory.Exists(tempPath))
DateTime dt = DateTime.Now;
if (date && !String.IsNullOrWhiteSpace(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt))
{ {
Directory.CreateDirectory(tempPath); uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
} }
// Create a stream dictionary // Copy the input stream to the output
Dictionary<string, Stream> dict = new Dictionary<string, Stream>(); byte[] ibuffer = new byte[_bufferSize];
dict.Add(rom.Name, inputStream); int ilen;
while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0)
// Now add the stream {
zipFile.CompressStreamDictionary(dict, tempFile); writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
inputStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(rom.CRC));
} }
// Otherwise, sort the input files and write out in the correct order // Otherwise, sort the input files and write out in the correct order
else else
{ {
// Open the old archive for reading // Open the old archive for reading
using (oldZipFile = new SevenZipExtractor(archiveFileName)) oldZipFile.ZipFileOpen(archiveFileName, -1, true);
{
// Map all inputs to index // Map all inputs to index
Dictionary<string, int> inputIndexMap = new Dictionary<string, int>(); Dictionary<string, int> inputIndexMap = new Dictionary<string, int>();
var oldZipFileContents = new List<string>();
for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{
oldZipFileContents.Add(oldZipFile.Filename(i));
}
// If the old one doesn't contain the new file, then add it // If the old one doesn't contain the new file, then add it
if (!oldZipFile.ArchiveFileNames.Contains(rom.Name.Replace('\\', '/'))) if (!oldZipFileContents.Contains(rom.Name.Replace('\\', '/')))
{ {
inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1); inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1);
} }
// Then add all of the old entries to it too // Then add all of the old entries to it too
for (int i = 0; i < oldZipFile.FilesCount; i++) for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{ {
inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); inputIndexMap.Add(oldZipFile.Filename(i), i);
} }
// If the number of entries is the same as the old archive, skip out // If the number of entries is the same as the old archive, skip out
if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) if (inputIndexMap.Keys.Count <= oldZipFile.LocalFilesCount())
{ {
success = true; success = true;
return success; return success;
} }
// Otherwise, process the old zipfile // Otherwise, process the old zipfile
zipFile = new SevenZipCompressor() zipFile.ZipFileCreate(tempFile);
{
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionLevel = CompressionLevel.Normal,
};
// Get the order for the entries with the new file // Get the order for the entries with the new file
List<string> keys = inputIndexMap.Keys.ToList(); List<string> keys = inputIndexMap.Keys.ToList();
@@ -458,36 +561,59 @@ namespace SabreTools.Library.FileTypes
// If we have the input file, add it now // If we have the input file, add it now
if (index < 0) if (index < 0)
{ {
// Create a stream dictionary // Open the input file for reading
Dictionary<string, Stream> dict = new Dictionary<string, Stream>(); ulong istreamSize = (ulong)(inputStream.Length);
dict.Add(rom.Name, inputStream);
// Now add the stream DateTime dt = DateTime.Now;
zipFile.CompressStreamDictionary(dict, tempFile); if (date && !String.IsNullOrWhiteSpace(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
inputStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(rom.CRC));
} }
// Otherwise, copy the file from the old archive // Otherwise, copy the file from the old archive
else else
{ {
Stream oldZipFileEntryStream = new MemoryStream(); // Instantiate the streams
oldZipFile.ExtractFile(index, oldZipFileEntryStream); oldZipFile.ZipFileOpenReadStream(index, out Stream zreadStream, out ulong istreamSize);
oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); zipFile.ZipFileOpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, 0, out writeStream);
// Create a stream dictionary // Copy the input stream to the output
Dictionary<string, Stream> dict = new Dictionary<string, Stream>(); byte[] ibuffer = new byte[_bufferSize];
dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); int ilen;
while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0)
// Now add the stream {
zipFile.CompressStreamDictionary(dict, tempFile); writeStream.Write(ibuffer, 0, ilen);
oldZipFileEntryStream.Dispose(); writeStream.Flush();
} }
// After the first file, make sure we're in append mode oldZipFile.ZipFileCloseReadStream();
zipFile.CompressionMode = CompressionMode.Append; zipFile.ZipFileCloseWriteStream(oldZipFile.CRC32(index));
} }
} }
} }
// Close the output zip file
zipFile.ZipFileClose();
oldZipFile.ZipFileClose();
success = true; success = true;
} }
catch (Exception ex) catch (Exception ex)
@@ -507,33 +633,6 @@ namespace SabreTools.Library.FileTypes
} }
File.Move(tempFile, archiveFileName); File.Move(tempFile, archiveFileName);
// Now make the file T7Z
// TODO: Add ACTUAL T7Z compatible code
BinaryWriter bw = new BinaryWriter(Utilities.TryOpenReadWrite(archiveFileName));
bw.Seek(0, SeekOrigin.Begin);
bw.Write(Constants.Torrent7ZipHeader);
bw.Seek(0, SeekOrigin.End);
using (oldZipFile = new SevenZipExtractor(Utilities.TryOpenReadWrite(archiveFileName)))
{
// Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4)
byte[] tempsig = Constants.Torrent7ZipSignature;
if (oldZipFile.FilesCount > 1)
{
tempsig[16] = 0x2;
}
else
{
tempsig[16] = 0;
}
bw.Write(tempsig);
bw.Flush();
bw.Dispose();
}
return true; return true;
} }
@@ -573,12 +672,13 @@ namespace SabreTools.Library.FileTypes
} }
// Get the output archive name from the first rebuild rom // Get the output archive name from the first rebuild rom
string archiveFileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".7z") ? "" : ".7z")); string archiveFileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".zip") ? "" : ".zip"));
// Set internal variables // Set internal variables
SevenZipBase.SetLibraryPath("7za.dll"); Stream writeStream = null;
SevenZipExtractor oldZipFile; SevenZ oldZipFile = new SevenZ();
SevenZipCompressor zipFile; SevenZ zipFile = new SevenZ();
ZipReturn zipReturn = ZipReturn.ZipGood;
try try
{ {
@@ -591,11 +691,7 @@ namespace SabreTools.Library.FileTypes
// If the archive doesn't exist, create it and put the single file // If the archive doesn't exist, create it and put the single file
if (!File.Exists(archiveFileName)) if (!File.Exists(archiveFileName))
{ {
zipFile = new SevenZipCompressor() zipReturn = zipFile.ZipFileCreate(tempFile);
{
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionLevel = CompressionLevel.Normal,
};
// Map all inputs to index // Map all inputs to index
Dictionary<string, int> inputIndexMap = new Dictionary<string, int>(); Dictionary<string, int> inputIndexMap = new Dictionary<string, int>();
@@ -608,42 +704,59 @@ namespace SabreTools.Library.FileTypes
List<string> keys = inputIndexMap.Keys.ToList(); List<string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TrrntZipStringCompare); keys.Sort(ZipFile.TrrntZipStringCompare);
// Create the temp directory
string tempPath = Path.Combine(outDir, Guid.NewGuid().ToString());
if (!Directory.Exists(tempPath))
{
Directory.CreateDirectory(tempPath);
}
// Now add all of the files in order // Now add all of the files in order
foreach (string key in keys) foreach (string key in keys)
{ {
string newkey = Path.Combine(tempPath, key); // Get the index mapped to the key
int index = inputIndexMap[key];
File.Move(inputFiles[inputIndexMap[key]], newkey); // Open the input file for reading
zipFile.CompressFiles(tempFile, newkey); Stream freadStream = Utilities.TryOpenRead(inputFiles[index]);
File.Move(newkey, inputFiles[inputIndexMap[key]]); ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length);
// After the first file, make sure we're in append mode DateTime dt = DateTime.Now;
zipFile.CompressionMode = CompressionMode.Append; if (date && !String.IsNullOrWhiteSpace(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
} }
Utilities.CleanDirectory(tempPath); // Copy the input stream to the output
Utilities.TryDeleteDirectory(tempPath); byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(roms[index].CRC));
}
} }
// Otherwise, sort the input files and write out in the correct order // Otherwise, sort the input files and write out in the correct order
else else
{ {
// Open the old archive for reading // Open the old archive for reading
using (oldZipFile = new SevenZipExtractor(archiveFileName)) oldZipFile.ZipFileOpen(archiveFileName, -1, true);
{
// Map all inputs to index // Map all inputs to index
Dictionary<string, int> inputIndexMap = new Dictionary<string, int>(); Dictionary<string, int> inputIndexMap = new Dictionary<string, int>();
for (int i = 0; i < inputFiles.Count; i++) for (int i = 0; i < inputFiles.Count; i++)
{ {
var oldZipFileContents = new List<string>();
for (int j = 0; j < oldZipFile.LocalFilesCount(); j++)
{
oldZipFileContents.Add(oldZipFile.Filename(j));
}
// If the old one contains the new file, then just skip out // If the old one contains the new file, then just skip out
if (oldZipFile.ArchiveFileNames.Contains(roms[i].Name.Replace('\\', '/'))) if (oldZipFileContents.Contains(roms[i].Name.Replace('\\', '/')))
{ {
continue; continue;
} }
@@ -652,24 +765,20 @@ namespace SabreTools.Library.FileTypes
} }
// Then add all of the old entries to it too // Then add all of the old entries to it too
for (int i = 0; i < oldZipFile.FilesCount; i++) for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{ {
inputIndexMap.Add(oldZipFile.ArchiveFileNames[i], i); inputIndexMap.Add(oldZipFile.Filename(i), i);
} }
// If the number of entries is the same as the old archive, skip out // If the number of entries is the same as the old archive, skip out
if (inputIndexMap.Keys.Count <= oldZipFile.FilesCount) if (inputIndexMap.Keys.Count <= oldZipFile.LocalFilesCount())
{ {
success = true; success = true;
return success; return success;
} }
// Otherwise, process the old zipfile // Otherwise, process the old zipfile
zipFile = new SevenZipCompressor() zipFile.ZipFileCreate(tempFile);
{
ArchiveFormat = OutArchiveFormat.SevenZip,
CompressionLevel = CompressionLevel.Normal,
};
// Get the order for the entries with the new file // Get the order for the entries with the new file
List<string> keys = inputIndexMap.Keys.ToList(); List<string> keys = inputIndexMap.Keys.ToList();
@@ -684,38 +793,58 @@ namespace SabreTools.Library.FileTypes
// If we have the input file, add it now // If we have the input file, add it now
if (index < 0) if (index < 0)
{ {
FileStream inputStream = Utilities.TryOpenRead(inputFiles[-index - 1]); // Open the input file for reading
Stream freadStream = Utilities.TryOpenRead(inputFiles[-index - 1]);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length);
// Create a stream dictionary DateTime dt = DateTime.Now;
Dictionary<string, Stream> dict = new Dictionary<string, Stream>(); if (date && !String.IsNullOrWhiteSpace(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt))
dict.Add(key, inputStream); {
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, 0, out writeStream);
}
// Now add the stream // Copy the input stream to the output
zipFile.CompressStreamDictionary(dict, tempFile); byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(roms[-index - 1].CRC));
} }
// Otherwise, copy the file from the old archive // Otherwise, copy the file from the old archive
else else
{ {
Stream oldZipFileEntryStream = new MemoryStream(); // Instantiate the streams
oldZipFile.ExtractFile(index, oldZipFileEntryStream); oldZipFile.ZipFileOpenReadStream(index, out Stream zreadStream, out ulong istreamSize);
oldZipFileEntryStream.Seek(0, SeekOrigin.Begin); zipFile.ZipFileOpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, 0, out writeStream);
// Create a stream dictionary // Copy the input stream to the output
Dictionary<string, Stream> dict = new Dictionary<string, Stream>(); byte[] ibuffer = new byte[_bufferSize];
dict.Add(oldZipFile.ArchiveFileNames[index], oldZipFileEntryStream); int ilen;
while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0)
// Now add the stream {
zipFile.CompressStreamDictionary(dict, tempFile); writeStream.Write(ibuffer, 0, ilen);
oldZipFileEntryStream.Dispose(); writeStream.Flush();
} }
// After the first file, make sure we're in append mode zipFile.ZipFileCloseWriteStream(oldZipFile.CRC32(index));
zipFile.CompressionMode = CompressionMode.Append;
} }
} }
} }
// Close the output zip file
zipFile.ZipFileClose();
oldZipFile.ZipFileClose();
success = true; success = true;
} }
catch (Exception ex) catch (Exception ex)
@@ -731,32 +860,6 @@ namespace SabreTools.Library.FileTypes
} }
File.Move(tempFile, archiveFileName); File.Move(tempFile, archiveFileName);
// Now make the file T7Z
// TODO: Add ACTUAL T7Z compatible code
BinaryWriter bw = new BinaryWriter(Utilities.TryOpenReadWrite(archiveFileName));
bw.Seek(0, SeekOrigin.Begin);
bw.Write(Constants.Torrent7ZipHeader);
bw.Seek(0, SeekOrigin.End);
using (oldZipFile = new SevenZipExtractor(Utilities.TryOpenReadWrite(archiveFileName)))
{
// Get the correct signature to use (Default 0, Unicode 1, SingleFile 2, StripFileNames 4)
byte[] tempsig = Constants.Torrent7ZipSignature;
if (oldZipFile.FilesCount > 1)
{
tempsig[16] = 0x2;
}
else
{
tempsig[16] = 0;
}
bw.Write(tempsig);
bw.Flush();
bw.Dispose();
}
return true; return true;
} }

View File

@@ -19,7 +19,7 @@ using Stream = System.IO.Stream;
#endif #endif
using Compress; using Compress;
using Compress.ZipFile; using Compress.ZipFile;
using SharpCompress.Readers; using NaturalSort;
namespace SabreTools.Library.FileTypes namespace SabreTools.Library.FileTypes
{ {
@@ -93,6 +93,7 @@ namespace SabreTools.Library.FileTypes
|| zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString()) || zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.PathSeparator.ToString())) || zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{ {
zf.ZipFileCloseReadStream();
continue; continue;
} }
@@ -286,6 +287,15 @@ namespace SabreTools.Library.FileTypes
for (int i = 0; i < zf.LocalFilesCount(); i++) for (int i = 0; i < zf.LocalFilesCount(); i++)
{ {
// If the entry is a directory (or looks like a directory), we don't want to open it
if (zf.IsDirectory(i)
|| zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
continue;
}
// Open the read stream // Open the read stream
zr = zf.ZipFileOpenReadStream(i, false, out Stream readStream, out ulong streamsize, out ushort cm); zr = zf.ZipFileOpenReadStream(i, false, out Stream readStream, out ulong streamsize, out ushort cm);
@@ -297,15 +307,6 @@ namespace SabreTools.Library.FileTypes
continue; continue;
} }
// If the entry ends with a directory separator, continue to the next item, if any
if (zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString())
|| zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
zr = zf.ZipFileCloseReadStream();
continue;
}
// If secure hashes are disabled, do a quickscan // If secure hashes are disabled, do a quickscan
if (omitFromScan == Hash.SecureHashes) if (omitFromScan == Hash.SecureHashes)
{ {
@@ -360,27 +361,36 @@ namespace SabreTools.Library.FileTypes
try try
{ {
SharpCompress.Archives.Zip.ZipArchive za = SharpCompress.Archives.Zip.ZipArchive.Open(this.Filename, new ReaderOptions { LeaveStreamOpen = false }); ZipFile zf = new ZipFile();
List<SharpCompress.Archives.Zip.ZipArchiveEntry> zipEntries = za.Entries.OrderBy(e => e.Key, new NaturalSort.NaturalReversedComparer()).ToList(); ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
string lastZipEntry = null; if (zr != ZipReturn.ZipGood)
foreach (SharpCompress.Archives.Zip.ZipArchiveEntry entry in zipEntries)
{ {
if (entry != null) throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
List<(string, bool)> zipEntries = new List<(string, bool)>();
for (int i = 0; i < zf.LocalFilesCount(); i++)
{
zipEntries.Add((zf.Filename(i), zf.IsDirectory(i)));
}
zipEntries = zipEntries.OrderBy(p => p.Item1, new NaturalReversedComparer()).ToList();
string lastZipEntry = null;
foreach ((string, bool) entry in zipEntries)
{ {
// If the current is a superset of last, we skip it // If the current is a superset of last, we skip it
if (lastZipEntry != null && lastZipEntry.StartsWith(entry.Key)) if (lastZipEntry != null && lastZipEntry.StartsWith(entry.Item1))
{ {
// No-op // No-op
} }
// If the entry is a directory, we add it // If the entry is a directory, we add it
else else
{ {
if (entry.IsDirectory) if (entry.Item2)
{ {
empties.Add(entry.Key); empties.Add(entry.Item1);
}
lastZipEntry = entry.Key;
} }
lastZipEntry = entry.Item1;
} }
} }
} }
@@ -397,7 +407,14 @@ namespace SabreTools.Library.FileTypes
/// </summary> /// </summary>
public override bool IsTorrent() public override bool IsTorrent()
{ {
throw new NotImplementedException(); ZipFile zf = new ZipFile();
ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
return zf.ZipStatus == ZipStatus.TrrntZip;
} }
#endregion #endregion
@@ -501,7 +518,7 @@ namespace SabreTools.Library.FileTypes
else else
{ {
// Open the old archive for reading // Open the old archive for reading
oldZipFile.ZipFileOpen(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); oldZipFile.ZipFileOpen(archiveFileName, -1, true);
// Map all inputs to index // Map all inputs to index
Dictionary<string, int> inputIndexMap = new Dictionary<string, int>(); Dictionary<string, int> inputIndexMap = new Dictionary<string, int>();
@@ -728,7 +745,7 @@ namespace SabreTools.Library.FileTypes
else else
{ {
// Open the old archive for reading // Open the old archive for reading
oldZipFile.ZipFileOpen(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true); oldZipFile.ZipFileOpen(archiveFileName, -1, true);
// Map all inputs to index // Map all inputs to index
Dictionary<string, int> inputIndexMap = new Dictionary<string, int>(); Dictionary<string, int> inputIndexMap = new Dictionary<string, int>();

View File

@@ -47,7 +47,6 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<None Remove="7za.dll" />
<None Remove="Skippers\a7800.xml" /> <None Remove="Skippers\a7800.xml" />
<None Remove="Skippers\fds.xml" /> <None Remove="Skippers\fds.xml" />
<None Remove="Skippers\lynx.xml" /> <None Remove="Skippers\lynx.xml" />
@@ -61,9 +60,6 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Content Include="7za.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="Skippers\a7800.xml"> <Content Include="Skippers\a7800.xml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content> </Content>
@@ -100,7 +96,7 @@
<PackageReference Include="AlphaFS" Version="2.2.6" /> <PackageReference Include="AlphaFS" Version="2.2.6" />
<PackageReference Include="Mono.Data.Sqlite.Portable" Version="1.0.3.5" /> <PackageReference Include="Mono.Data.Sqlite.Portable" Version="1.0.3.5" />
<PackageReference Include="SevenZipSharp.Net45" Version="1.0.19" /> <PackageReference Include="SevenZipSharp.Net45" Version="1.0.19" />
<PackageReference Include="SharpCompress" Version="0.23.0" /> <PackageReference Include="SharpCompress" Version="0.24.0" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" /> <PackageReference Include="System.ValueTuple" Version="4.5.0" />
</ItemGroup> </ItemGroup>