From 8340d1edd6c998cac4869be27b1e5ef2f5f4e6cf Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Sun, 28 Apr 2013 11:25:37 +0100 Subject: [PATCH] Port from HG --- .../Archive/SevenZip/SevenZipArchive.cs | 59 +- .../Common/SevenZip/ArchiveDatabase.cs | 161 ++ .../Common/SevenZip/ArchiveReader.cs | 1277 ++++++++++++ .../Common/SevenZip/ArchiveWriter.cs | 1783 +++++++++++++++++ SharpCompress/Common/SevenZip/CBindPair.cs | 8 + SharpCompress/Common/SevenZip/CCoderInfo.cs | 10 + SharpCompress/Common/SevenZip/CFileItem.cs | 34 + SharpCompress/Common/SevenZip/CFolder.cs | 136 ++ SharpCompress/Common/SevenZip/CMethodId.cs | 55 + .../Common/SevenZip/CStreamSwitch.cs | 62 + SharpCompress/Common/SevenZip/DataReader.cs | 168 ++ .../Common/SevenZip/SevenZipEntry.cs | 4 +- .../Common/SevenZip/SevenZipFilePart.cs | 83 +- .../Compressor/LZMA/AesDecoderStream.cs | 228 +++ .../Compressor/LZMA/Bcj2DecoderStream.cs | 226 +++ SharpCompress/Compressor/LZMA/BitVector.cs | 87 + SharpCompress/Compressor/LZMA/CRC.cs | 118 +- SharpCompress/Compressor/LZMA/CRC2.cs | 53 + .../Compressor/LZMA/DecoderStream.cs | 173 ++ SharpCompress/Compressor/LZMA/Log.cs | 91 + SharpCompress/Compressor/LZMA/Registry.cs | 52 + .../LZMA/Utilites/CrcBuilderStream.cs | 197 ++ .../LZMA/Utilites/CrcCheckStream.cs | 114 ++ .../LZMA/Utilites/IPasswordProvider.cs | 7 + .../LZMA/Utilites/SyncStreamView.cs | 108 + .../LZMA/Utilites/UnpackSubStream.cs | 96 + .../Compressor/LZMA/Utilites/Utils.cs | 80 + SharpCompress/SharpCompress.3.5.csproj | 28 +- .../SharpCompress.Silverlight.csproj | 27 +- SharpCompress/SharpCompress.WP7.csproj | 27 +- SharpCompress/SharpCompress.csproj | 68 +- 31 files changed, 5514 insertions(+), 106 deletions(-) create mode 100644 SharpCompress/Common/SevenZip/ArchiveDatabase.cs create mode 100644 SharpCompress/Common/SevenZip/ArchiveReader.cs create mode 100644 SharpCompress/Common/SevenZip/ArchiveWriter.cs create mode 100644 SharpCompress/Common/SevenZip/CBindPair.cs create mode 100644 SharpCompress/Common/SevenZip/CCoderInfo.cs create mode 100644 SharpCompress/Common/SevenZip/CFileItem.cs create mode 100644 SharpCompress/Common/SevenZip/CFolder.cs create mode 100644 SharpCompress/Common/SevenZip/CMethodId.cs create mode 100644 SharpCompress/Common/SevenZip/CStreamSwitch.cs create mode 100644 SharpCompress/Common/SevenZip/DataReader.cs create mode 100644 SharpCompress/Compressor/LZMA/AesDecoderStream.cs create mode 100644 SharpCompress/Compressor/LZMA/Bcj2DecoderStream.cs create mode 100644 SharpCompress/Compressor/LZMA/BitVector.cs create mode 100644 SharpCompress/Compressor/LZMA/CRC2.cs create mode 100644 SharpCompress/Compressor/LZMA/DecoderStream.cs create mode 100644 SharpCompress/Compressor/LZMA/Log.cs create mode 100644 SharpCompress/Compressor/LZMA/Registry.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/CrcBuilderStream.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/CrcCheckStream.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/IPasswordProvider.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/SyncStreamView.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/UnpackSubStream.cs create mode 100644 SharpCompress/Compressor/LZMA/Utilites/Utils.cs diff --git a/SharpCompress/Archive/SevenZip/SevenZipArchive.cs b/SharpCompress/Archive/SevenZip/SevenZipArchive.cs index 7cd2546b..b2c9ede3 100644 --- a/SharpCompress/Archive/SevenZip/SevenZipArchive.cs +++ b/SharpCompress/Archive/SevenZip/SevenZipArchive.cs @@ -11,7 +11,7 @@ namespace SharpCompress.Archive.SevenZip { public class SevenZipArchive : AbstractArchive { - private SevenZipHeaderFactory factory; + private ArchiveDatabase database; #if !PORTABLE /// /// Constructor expects a filepath to an existing file. @@ -130,19 +130,24 @@ namespace SharpCompress.Archive.SevenZip { var stream = volumes.Single().Stream; LoadFactory(stream); - for (int i = 0; i < factory.Entries.Length; i++) + for (int i = 0; i < database.Files.Count; i++) { - var file = factory.Entries[i]; - yield return new SevenZipArchiveEntry(this, new SevenZipFilePart(factory, i, file, stream)); + var file = database.Files[i]; + if (!file.IsDir) + { + yield return new SevenZipArchiveEntry(this, new SevenZipFilePart(stream, database, i, file)); + } } } private void LoadFactory(Stream stream) { - if (factory == null) + if (database == null) { stream.Position = 0; - factory = new SevenZipHeaderFactory(stream); + var reader = new ArchiveReader(); + reader.Open(stream); + database = reader.ReadDatabase(null); } } @@ -151,13 +156,20 @@ namespace SharpCompress.Archive.SevenZip { try { - return SevenZipHeaderFactory.SignatureMatch(stream); + return SignatureMatch(stream); } catch { return false; } } + private static readonly byte[] SIGNATURE = new byte[] { (byte)'7', (byte)'z', 0xBC, 0xAF, 0x27, 0x1C }; + private static bool SignatureMatch(Stream stream) + { + BinaryReader reader = new BinaryReader(stream); + byte[] signatureBytes = reader.ReadBytes(6); + return signatureBytes.BinaryEquals(SIGNATURE); + } protected override IReader CreateReaderForSolidExtraction() { @@ -168,15 +180,16 @@ namespace SharpCompress.Archive.SevenZip { get { - return Entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Header.Folder).Count() > 1; + return Entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Folder).Count() > 1; } } private class SevenZipReader : AbstractReader { private readonly SevenZipArchive archive; - private Folder currentFolder; + private CFolder currentFolder; private Stream currentStream; + private CFileItem currentItem; internal SevenZipReader(SevenZipArchive archive) : base(Options.KeepStreamsOpen, ArchiveType.SevenZip) @@ -198,26 +211,28 @@ namespace SharpCompress.Archive.SevenZip { yield return dir; } - foreach (var group in entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Header.Folder)) + foreach (var group in entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Folder)) { - currentFolder = group.Key; - currentStream = currentFolder.GetStream(); - foreach (var entry in group.OrderBy(x => x.FilePart.Header.FolderOffset)) - { - if (currentStream.Position != (long)entry.FilePart.Header.FolderOffset) - { - throw new InvalidFormatException("Unexpected SevenZip folder offset"); - } + currentFolder = group.Key; + if (group.Key == null) + { + currentStream = Stream.Null; + } + else + { + currentStream = archive.database.GetFolderStream(stream, currentFolder, null); + } + foreach (var entry in group) + { + currentItem = entry.FilePart.Header; yield return entry; - } + } } } protected override EntryStream GetEntryStream() { - return new EntryStream(new ReadOnlySubStream(currentStream, - (long) - Entry.Parts.Cast().Single().Header.UnpackedStream.UnpackedSize)); + return new EntryStream(new ReadOnlySubStream(currentStream, currentItem.Size)); } } } diff --git a/SharpCompress/Common/SevenZip/ArchiveDatabase.cs b/SharpCompress/Common/SevenZip/ArchiveDatabase.cs new file mode 100644 index 00000000..d119f7b5 --- /dev/null +++ b/SharpCompress/Common/SevenZip/ArchiveDatabase.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SharpCompress.Compressor.LZMA; +using SharpCompress.Compressor.LZMA.Utilites; + +namespace SharpCompress.Common.SevenZip +{ + public class ArchiveDatabase + { + internal byte MajorVersion; + internal byte MinorVersion; + internal long StartPositionAfterHeader; + internal long DataStartPosition; + + internal List PackSizes = new List(); + internal List PackCRCs = new List(); + internal List Folders = new List(); + internal List NumUnpackStreamsVector; + internal List Files = new List(); + + internal List PackStreamStartPositions = new List(); + internal List FolderStartFileIndex = new List(); + internal List FileIndexToFolderIndexMap = new List(); + + internal void Clear() + { + PackSizes.Clear(); + PackCRCs.Clear(); + Folders.Clear(); + NumUnpackStreamsVector = null; + Files.Clear(); + + PackStreamStartPositions.Clear(); + FolderStartFileIndex.Clear(); + FileIndexToFolderIndexMap.Clear(); + } + + internal bool IsEmpty() + { + return PackSizes.Count == 0 + && PackCRCs.Count == 0 + && Folders.Count == 0 + && NumUnpackStreamsVector.Count == 0 + && Files.Count == 0; + } + + private void FillStartPos() + { + PackStreamStartPositions.Clear(); + + long startPos = 0; + for(int i = 0; i < PackSizes.Count; i++) + { + PackStreamStartPositions.Add(startPos); + startPos += PackSizes[i]; + } + } + + private void FillFolderStartFileIndex() + { + FolderStartFileIndex.Clear(); + FileIndexToFolderIndexMap.Clear(); + + int folderIndex = 0; + int indexInFolder = 0; + for(int i = 0; i < Files.Count; i++) + { + CFileItem file = Files[i]; + + bool emptyStream = !file.HasStream; + + if(emptyStream && indexInFolder == 0) + { + FileIndexToFolderIndexMap.Add(-1); + continue; + } + + if(indexInFolder == 0) + { + // v3.13 incorrectly worked with empty folders + // v4.07: Loop for skipping empty folders + for(; ; ) + { + if(folderIndex >= Folders.Count) + throw new InvalidOperationException(); + + FolderStartFileIndex.Add(i); // check it + + if(NumUnpackStreamsVector[folderIndex] != 0) + break; + + folderIndex++; + } + } + + FileIndexToFolderIndexMap.Add(folderIndex); + + if(emptyStream) + continue; + + indexInFolder++; + + if(indexInFolder >= NumUnpackStreamsVector[folderIndex]) + { + folderIndex++; + indexInFolder = 0; + } + } + } + + public void Fill() + { + FillStartPos(); + FillFolderStartFileIndex(); + } + + internal long GetFolderStreamPos(CFolder folder, int indexInFolder) + { + int index = folder.FirstPackStreamId + indexInFolder; + return DataStartPosition + PackStreamStartPositions[index]; + } + + internal long GetFolderFullPackSize(int folderIndex) + { + int packStreamIndex = Folders[folderIndex].FirstPackStreamId; + CFolder folder = Folders[folderIndex]; + + long size = 0; + for(int i = 0; i < folder.PackStreams.Count; i++) + size += PackSizes[packStreamIndex + i]; + + return size; + } + + internal Stream GetFolderStream(Stream stream, CFolder folder, IPasswordProvider pw) + { + int packStreamIndex = folder.FirstPackStreamId; + long folderStartPackPos = GetFolderStreamPos(folder, 0); + List packSizes = new List(); + for (int j = 0; j < folder.PackStreams.Count; j++) + packSizes.Add(PackSizes[packStreamIndex + j]); + + return DecoderStreamHelper.CreateDecoderStream(stream, folderStartPackPos, packSizes.ToArray(), folder, pw); + } + + private long GetFolderPackStreamSize(int folderIndex, int streamIndex) + { + return PackSizes[Folders[folderIndex].FirstPackStreamId + streamIndex]; + } + + private long GetFilePackSize(int fileIndex) + { + int folderIndex = FileIndexToFolderIndexMap[fileIndex]; + if(folderIndex != -1) + if(FolderStartFileIndex[folderIndex] == fileIndex) + return GetFolderFullPackSize(folderIndex); + return 0; + } + } +} diff --git a/SharpCompress/Common/SevenZip/ArchiveReader.cs b/SharpCompress/Common/SevenZip/ArchiveReader.cs new file mode 100644 index 00000000..630cb876 --- /dev/null +++ b/SharpCompress/Common/SevenZip/ArchiveReader.cs @@ -0,0 +1,1277 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using SharpCompress.Compressor.LZMA; +using SharpCompress.Compressor.LZMA.Utilites; +using SharpCompress.IO; +using CRC = ManagedLzma.LZMA.Master.SevenZip.CRC; + +namespace SharpCompress.Common.SevenZip +{ + public class ArchiveReader + { + internal Stream _stream; + internal Stack _readerStack = new Stack(); + internal DataReader _currentReader; + internal long _streamOrigin; + internal long _streamEnding; + internal byte[] _header; + + private Dictionary _cachedStreams = new Dictionary(); + + internal void AddByteStream(byte[] buffer, int offset, int length) + { + _readerStack.Push(_currentReader); + _currentReader = new DataReader(buffer, offset, length); + } + + internal void DeleteByteStream() + { + _currentReader = _readerStack.Pop(); + } + + #region Private Methods - Data Reader + + internal Byte ReadByte() + { + return _currentReader.ReadByte(); + } + + private void ReadBytes(byte[] buffer, int offset, int length) + { + _currentReader.ReadBytes(buffer, offset, length); + } + + private ulong ReadNumber() + { + return _currentReader.ReadNumber(); + } + + internal int ReadNum() + { + return _currentReader.ReadNum(); + } + + private uint ReadUInt32() + { + return _currentReader.ReadUInt32(); + } + + private ulong ReadUInt64() + { + return _currentReader.ReadUInt64(); + } + + private BlockType? ReadId() + { + ulong id = _currentReader.ReadNumber(); + if(id > 25) + return null; + + Log.WriteLine("ReadId: {0}", (BlockType)id); + return (BlockType)id; + } + + private void SkipData(long size) + { + _currentReader.SkipData(size); + } + + private void SkipData() + { + _currentReader.SkipData(); + } + + private void WaitAttribute(BlockType attribute) + { + for(; ; ) + { + BlockType? type = ReadId(); + if(type == attribute) + return; + if(type == BlockType.End) + throw new InvalidOperationException(); + SkipData(); + } + } + + private void ReadArchiveProperties() + { + while(ReadId() != BlockType.End) + SkipData(); + } + + #endregion + + #region Private Methods - Reader Utilities + + private BitVector ReadBitVector(int length) + { + var bits = new BitVector(length); + + byte data = 0; + byte mask = 0; + + for(int i = 0; i < length; i++) + { + if(mask == 0) + { + data = ReadByte(); + mask = 0x80; + } + + if((data & mask) != 0) + bits.SetBit(i); + + mask >>= 1; + } + + return bits; + } + + private BitVector ReadOptionalBitVector(int length) + { + byte allTrue = ReadByte(); + if(allTrue != 0) + return new BitVector(length, true); + + return ReadBitVector(length); + } + + private void ReadNumberVector(List dataVector, int numFiles, Action action) + { + var defined = ReadOptionalBitVector(numFiles); + + using(CStreamSwitch streamSwitch = new CStreamSwitch()) + { + streamSwitch.Set(this, dataVector); + + for(int i = 0; i < numFiles; i++) + { + if(defined[i]) + action(i, checked((long)ReadUInt64())); + else + action(i, null); + } + } + } + + private DateTime TranslateTime(long time) + { + // FILETIME = 100-nanosecond intervals since January 1, 1601 (UTC) + return DateTime.FromFileTimeUtc(time); + } + + private DateTime? TranslateTime(long? time) + { + if(time.HasValue) + return TranslateTime(time.Value); + else + return null; + } + + private void ReadDateTimeVector(List dataVector, int numFiles, Action action) + { + ReadNumberVector(dataVector, numFiles, (index, value) => action(index, TranslateTime(value))); + } + + private void ReadAttributeVector(List dataVector, int numFiles, Action action) + { + BitVector boolVector = ReadOptionalBitVector(numFiles); + using(var streamSwitch = new CStreamSwitch()) + { + streamSwitch.Set(this, dataVector); + for(int i = 0; i < numFiles; i++) + { + if(boolVector[i]) + action(i, ReadUInt32()); + else + action(i, null); + } + } + } + + #endregion + + #region Private Methods + + private void GetNextFolderItem(CFolder folder) + { + Log.WriteLine("-- GetNextFolderItem --"); + Log.PushIndent(); + try + { + int numCoders = ReadNum(); + Log.WriteLine("NumCoders: " + numCoders); + + folder.Coders = new List(numCoders); + int numInStreams = 0; + int numOutStreams = 0; + for(int i = 0; i < numCoders; i++) + { + Log.WriteLine("-- Coder --"); + Log.PushIndent(); + try + { + CCoderInfo coder = new CCoderInfo(); + folder.Coders.Add(coder); + + byte mainByte = ReadByte(); + int idSize = (mainByte & 0xF); + byte[] longID = new byte[idSize]; + ReadBytes(longID, 0, idSize); + Log.WriteLine("MethodId: " + String.Join("", Enumerable.Range(0, idSize).Select(x => longID[x].ToString("x2")).ToArray())); + if(idSize > 8) + throw new NotSupportedException(); + ulong id = 0; + for(int j = 0; j < idSize; j++) + id |= (ulong)longID[idSize - 1 - j] << (8 * j); + coder.MethodId = new CMethodId(id); + + if((mainByte & 0x10) != 0) + { + coder.NumInStreams = ReadNum(); + coder.NumOutStreams = ReadNum(); + Log.WriteLine("Complex Stream (In: " + coder.NumInStreams + " - Out: " + coder.NumOutStreams + ")"); + } + else + { + Log.WriteLine("Simple Stream (In: 1 - Out: 1)"); + coder.NumInStreams = 1; + coder.NumOutStreams = 1; + } + + if((mainByte & 0x20) != 0) + { + int propsSize = ReadNum(); + coder.Props = new byte[propsSize]; + ReadBytes(coder.Props, 0, propsSize); + Log.WriteLine("Settings: " + String.Join("", coder.Props.Select(bt => bt.ToString("x2")).ToArray())); + } + + if((mainByte & 0x80) != 0) + throw new NotSupportedException(); + + numInStreams += coder.NumInStreams; + numOutStreams += coder.NumOutStreams; + } + finally { Log.PopIndent(); } + } + + int numBindPairs = numOutStreams - 1; + folder.BindPairs = new List(numBindPairs); + Log.WriteLine("BindPairs: " + numBindPairs); + Log.PushIndent(); + for(int i = 0; i < numBindPairs; i++) + { + CBindPair bp = new CBindPair(); + bp.InIndex = ReadNum(); + bp.OutIndex = ReadNum(); + folder.BindPairs.Add(bp); + Log.WriteLine("#" + i + " - In: " + bp.InIndex + " - Out: " + bp.OutIndex); + } + Log.PopIndent(); + + if(numInStreams < numBindPairs) + throw new NotSupportedException(); + + int numPackStreams = numInStreams - numBindPairs; + //folder.PackStreams.Reserve(numPackStreams); + if(numPackStreams == 1) + { + for(int i = 0; i < numInStreams; i++) + { + if(folder.FindBindPairForInStream(i) < 0) + { + Log.WriteLine("Single PackStream: #" + i); + folder.PackStreams.Add(i); + break; + } + } + + if(folder.PackStreams.Count != 1) + throw new NotSupportedException(); + } + else + { + Log.WriteLine("Multiple PackStreams ..."); + Log.PushIndent(); + for(int i = 0; i < numPackStreams; i++) + { + var num = ReadNum(); + Log.WriteLine("#" + i + " - " + num); + folder.PackStreams.Add(num); + } + Log.PopIndent(); + } + } + finally + { + Log.PopIndent(); + } + } + + private List ReadHashDigests(int count) + { + Log.Write("ReadHashDigests:"); + + var defined = ReadOptionalBitVector(count); + var digests = new List(count); + for(int i = 0; i < count; i++) + { + if(defined[i]) + { + uint crc = ReadUInt32(); + Log.Write(" " + crc.ToString("x8")); + digests.Add(crc); + } + else + { + Log.Write(" ########"); + digests.Add(null); + } + } + + Log.WriteLine(); + return digests; + } + + private void ReadPackInfo(out long dataOffset, out List packSizes, out List packCRCs) + { + Log.WriteLine("-- ReadPackInfo --"); + Log.PushIndent(); + try + { + packCRCs = null; + + dataOffset = checked((long)ReadNumber()); + Log.WriteLine("DataOffset: " + dataOffset); + + int numPackStreams = ReadNum(); + Log.WriteLine("NumPackStreams: " + numPackStreams); + + WaitAttribute(BlockType.Size); + packSizes = new List(numPackStreams); + Log.Write("Sizes:"); + for(int i = 0; i < numPackStreams; i++) + { + var size = checked((long)ReadNumber()); + Log.Write(" " + size); + packSizes.Add(size); + } + Log.WriteLine(); + + BlockType? type; + for(; ; ) + { + type = ReadId(); + if(type == BlockType.End) + break; + if(type == BlockType.CRC) + { + packCRCs = ReadHashDigests(numPackStreams); + continue; + } + SkipData(); + } + + if(packCRCs == null) + { + packCRCs = new List(numPackStreams); + for(int i = 0; i < numPackStreams; i++) + packCRCs.Add(null); + } + } + finally { Log.PopIndent(); } + } + + private void ReadUnpackInfo(List dataVector, out List folders) + { + Log.WriteLine("-- ReadUnpackInfo --"); + Log.PushIndent(); + try + { + WaitAttribute(BlockType.Folder); + int numFolders = ReadNum(); + Log.WriteLine("NumFolders: {0}", numFolders); + + using(CStreamSwitch streamSwitch = new CStreamSwitch()) + { + streamSwitch.Set(this, dataVector); + //folders.Clear(); + //folders.Reserve(numFolders); + folders = new List(numFolders); + int index = 0; + for(int i = 0; i < numFolders; i++) + { + var f = new CFolder { FirstPackStreamId = index }; + folders.Add(f); + GetNextFolderItem(f); + index += f.PackStreams.Count; + } + } + + WaitAttribute(BlockType.CodersUnpackSize); + + Log.WriteLine("UnpackSizes:"); + for(int i = 0; i < numFolders; i++) + { + CFolder folder = folders[i]; + Log.Write(" #" + i + ":"); + int numOutStreams = folder.GetNumOutStreams(); + for(int j = 0; j < numOutStreams; j++) + { + long size = checked((long)ReadNumber()); + Log.Write(" " + size); + folder.UnpackSizes.Add(size); + } + Log.WriteLine(); + } + + for(; ; ) + { + BlockType? type = ReadId(); + if(type == BlockType.End) + return; + + if(type == BlockType.CRC) + { + List crcs = ReadHashDigests(numFolders); + for(int i = 0; i < numFolders; i++) + folders[i].UnpackCRC = crcs[i]; + continue; + } + + SkipData(); + } + } + finally { Log.PopIndent(); } + } + + private void ReadSubStreamsInfo(List folders, out List numUnpackStreamsInFolders, out List unpackSizes, out List digests) + { + Log.WriteLine("-- ReadSubStreamsInfo --"); + Log.PushIndent(); + try + { + numUnpackStreamsInFolders = null; + + BlockType? type; + for(; ; ) + { + type = ReadId(); + if(type == BlockType.NumUnpackStream) + { + numUnpackStreamsInFolders = new List(folders.Count); + Log.Write("NumUnpackStreams:"); + for(int i = 0; i < folders.Count; i++) + { + var num = ReadNum(); + Log.Write(" " + num); + numUnpackStreamsInFolders.Add(num); + } + Log.WriteLine(); + continue; + } + if(type == BlockType.CRC || type == BlockType.Size) + break; + if(type == BlockType.End) + break; + SkipData(); + } + + if(numUnpackStreamsInFolders == null) + { + numUnpackStreamsInFolders = new List(folders.Count); + for(int i = 0; i < folders.Count; i++) + numUnpackStreamsInFolders.Add(1); + } + + unpackSizes = new List(folders.Count); + for(int i = 0; i < numUnpackStreamsInFolders.Count; i++) + { + // v3.13 incorrectly worked with empty folders + // v4.07: we check that folder is empty + int numSubstreams = numUnpackStreamsInFolders[i]; + if(numSubstreams == 0) + continue; + + Log.Write("#{0} StreamSizes:", i); + long sum = 0; + for(int j = 1; j < numSubstreams; j++) + { + if(type == BlockType.Size) + { + long size = checked((long)ReadNumber()); + Log.Write(" " + size); + unpackSizes.Add(size); + sum += size; + } + } + unpackSizes.Add(folders[i].GetUnpackSize() - sum); + Log.WriteLine(" - rest: " + unpackSizes.Last()); + } + if(type == BlockType.Size) + type = ReadId(); + + int numDigests = 0; + int numDigestsTotal = 0; + for(int i = 0; i < folders.Count; i++) + { + int numSubstreams = numUnpackStreamsInFolders[i]; + if(numSubstreams != 1 || !folders[i].UnpackCRCDefined) + numDigests += numSubstreams; + numDigestsTotal += numSubstreams; + } + + digests = null; + + for(; ; ) + { + if(type == BlockType.CRC) + { + digests = new List(numDigestsTotal); + + List digests2 = ReadHashDigests(numDigests); + + int digestIndex = 0; + for(int i = 0; i < folders.Count; i++) + { + int numSubstreams = numUnpackStreamsInFolders[i]; + CFolder folder = folders[i]; + if(numSubstreams == 1 && folder.UnpackCRCDefined) + { + digests.Add(folder.UnpackCRC.Value); + } + else + { + for(int j = 0; j < numSubstreams; j++, digestIndex++) + digests.Add(digests2[digestIndex]); + } + } + + if(digestIndex != numDigests || numDigestsTotal != digests.Count) + System.Diagnostics.Debugger.Break(); + } + else if(type == BlockType.End) + { + if(digests == null) + { + digests = new List(numDigestsTotal); + for(int i = 0; i < numDigestsTotal; i++) + digests.Add(null); + } + return; + } + else + { + SkipData(); + } + + type = ReadId(); + } + } + finally { Log.PopIndent(); } + } + + private void ReadStreamsInfo( + List dataVector, + out long dataOffset, + out List packSizes, + out List packCRCs, + out List folders, + out List numUnpackStreamsInFolders, + out List unpackSizes, + out List digests) + { + Log.WriteLine("-- ReadStreamsInfo --"); + Log.PushIndent(); + try + { + dataOffset = long.MinValue; + packSizes = null; + packCRCs = null; + folders = null; + numUnpackStreamsInFolders = null; + unpackSizes = null; + digests = null; + + for(; ; ) + { + switch(ReadId()) + { + case BlockType.End: + return; + case BlockType.PackInfo: + ReadPackInfo(out dataOffset, out packSizes, out packCRCs); + break; + case BlockType.UnpackInfo: + ReadUnpackInfo(dataVector, out folders); + break; + case BlockType.SubStreamsInfo: + ReadSubStreamsInfo(folders, out numUnpackStreamsInFolders, out unpackSizes, out digests); + break; + default: + throw new InvalidOperationException(); + } + } + } + finally { Log.PopIndent(); } + } + + private List ReadAndDecodePackedStreams(long baseOffset, IPasswordProvider pass) + { + Log.WriteLine("-- ReadAndDecodePackedStreams --"); + Log.PushIndent(); + try + { + long dataStartPos; + List packSizes; + List packCRCs; + List folders; + List numUnpackStreamsInFolders; + List unpackSizes; + List digests; + + ReadStreamsInfo(null, + out dataStartPos, + out packSizes, + out packCRCs, + out folders, + out numUnpackStreamsInFolders, + out unpackSizes, + out digests); + + dataStartPos += baseOffset; + + var dataVector = new List(folders.Count); + int packIndex = 0; + foreach(var folder in folders) + { + long oldDataStartPos = dataStartPos; + long[] myPackSizes = new long[folder.PackStreams.Count]; + for(int i = 0; i < myPackSizes.Length; i++) + { + long packSize = packSizes[packIndex + i]; + myPackSizes[i] = packSize; + dataStartPos += packSize; + } + + var outStream = DecoderStreamHelper.CreateDecoderStream(_stream, oldDataStartPos, myPackSizes, folder, pass); + + int unpackSize = checked((int)folder.GetUnpackSize()); + byte[] data = new byte[unpackSize]; + outStream.ReadExact(data, 0, data.Length); + if(outStream.ReadByte() >= 0) + throw new InvalidOperationException("Decoded stream is longer than expected."); + dataVector.Add(data); + + if(folder.UnpackCRCDefined) + if(CRC.Finish(CRC.Update(CRC.kInitCRC, data, 0, unpackSize)) != folder.UnpackCRC) + throw new InvalidOperationException("Decoded stream does not match expected CRC."); + } + return dataVector; + } + finally { Log.PopIndent(); } + } + + private void ReadHeader(ArchiveDatabase db, IPasswordProvider getTextPassword) + { + Log.WriteLine("-- ReadHeader --"); + Log.PushIndent(); + try + { + BlockType? type = ReadId(); + + if(type == BlockType.ArchiveProperties) + { + ReadArchiveProperties(); + type = ReadId(); + } + + List dataVector = null; + if(type == BlockType.AdditionalStreamsInfo) + { + dataVector = ReadAndDecodePackedStreams(db.StartPositionAfterHeader, getTextPassword); + type = ReadId(); + } + + List unpackSizes; + List digests; + + if(type == BlockType.MainStreamsInfo) + { + ReadStreamsInfo(dataVector, + out db.DataStartPosition, + out db.PackSizes, + out db.PackCRCs, + out db.Folders, + out db.NumUnpackStreamsVector, + out unpackSizes, + out digests); + + db.DataStartPosition += db.StartPositionAfterHeader; + type = ReadId(); + } + else + { + unpackSizes = new List(db.Folders.Count); + digests = new List(db.Folders.Count); + db.NumUnpackStreamsVector = new List(db.Folders.Count); + for(int i = 0; i < db.Folders.Count; i++) + { + var folder = db.Folders[i]; + unpackSizes.Add(folder.GetUnpackSize()); + digests.Add(folder.UnpackCRC); + db.NumUnpackStreamsVector.Add(1); + } + } + + db.Files.Clear(); + + if(type == BlockType.End) + return; + + if(type != BlockType.FilesInfo) + throw new InvalidOperationException(); + + int numFiles = ReadNum(); + Log.WriteLine("NumFiles: " + numFiles); + db.Files = new List(numFiles); + for(int i = 0; i < numFiles; i++) + db.Files.Add(new CFileItem()); + + BitVector emptyStreamVector = new BitVector(numFiles); + BitVector emptyFileVector = null; + BitVector antiFileVector = null; + int numEmptyStreams = 0; + + for(; ; ) + { + type = ReadId(); + if(type == BlockType.End) + break; + + long size = checked((long)ReadNumber()); // TODO: throw invalid data on negative + int oldPos = _currentReader.Offset; + switch(type) + { + case BlockType.Name: + using(var streamSwitch = new CStreamSwitch()) + { + streamSwitch.Set(this, dataVector); + Log.Write("FileNames:"); + for(int i = 0; i < db.Files.Count; i++) + { + db.Files[i].Name = _currentReader.ReadString(); + Log.Write(" " + db.Files[i].Name); + } + Log.WriteLine(); + } + break; + case BlockType.WinAttributes: + Log.Write("WinAttributes:"); + ReadAttributeVector(dataVector, numFiles, delegate(int i, uint? attr) { + db.Files[i].Attrib = attr; + Log.Write(" " + (attr.HasValue ? attr.Value.ToString("x8") : "n/a")); + }); + Log.WriteLine(); + break; + case BlockType.EmptyStream: + emptyStreamVector = ReadBitVector(numFiles); + + Log.Write("EmptyStream: "); + for(int i = 0; i < emptyStreamVector.Length; i++) + { + if(emptyStreamVector[i]) + { + Log.Write("x"); + numEmptyStreams++; + } + else + { + Log.Write("."); + } + } + Log.WriteLine(); + + emptyFileVector = new BitVector(numEmptyStreams); + antiFileVector = new BitVector(numEmptyStreams); + break; + case BlockType.EmptyFile: + emptyFileVector = ReadBitVector(numEmptyStreams); + Log.Write("EmptyFile: "); + for(int i = 0; i < numEmptyStreams; i++) + Log.Write(emptyFileVector[i] ? "x" : "."); + Log.WriteLine(); + break; + case BlockType.Anti: + antiFileVector = ReadBitVector(numEmptyStreams); + Log.Write("Anti: "); + for(int i = 0; i < numEmptyStreams; i++) + Log.Write(antiFileVector[i] ? "x" : "."); + Log.WriteLine(); + break; + case BlockType.StartPos: + Log.Write("StartPos:"); + ReadNumberVector(dataVector, numFiles, delegate(int i, long? startPos) { + db.Files[i].StartPos = startPos; + Log.Write(" " + (startPos.HasValue ? startPos.Value.ToString() : "n/a")); + }); + Log.WriteLine(); + break; + case BlockType.CTime: + Log.Write("CTime:"); + ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time) { + db.Files[i].CTime = time; + Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a")); + }); + Log.WriteLine(); + break; + case BlockType.ATime: + Log.Write("ATime:"); + ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time) { + db.Files[i].ATime = time; + Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a")); + }); + Log.WriteLine(); + break; + case BlockType.MTime: + Log.Write("MTime:"); + ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time) { + db.Files[i].MTime = time; + Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a")); + }); + Log.WriteLine(); + break; + case BlockType.Dummy: + Log.Write("Dummy: " + size); + for(long j = 0; j < size; j++) + if(ReadByte() != 0) + throw new InvalidOperationException(); + break; + default: + SkipData(size); + break; + } + + // since 0.3 record sizes must be correct + bool checkRecordsSize = (db.MajorVersion > 0 || db.MinorVersion > 2); + if(checkRecordsSize && _currentReader.Offset - oldPos != size) + throw new InvalidOperationException(); + } + + int emptyFileIndex = 0; + int sizeIndex = 0; + for(int i = 0; i < numFiles; i++) + { + CFileItem file = db.Files[i]; + file.HasStream = !emptyStreamVector[i]; + if(file.HasStream) + { + file.IsDir = false; + file.IsAnti = false; + file.Size = unpackSizes[sizeIndex]; + file.Crc = digests[sizeIndex]; + sizeIndex++; + } + else + { + file.IsDir = !emptyFileVector[emptyFileIndex]; + file.IsAnti = antiFileVector[emptyFileIndex]; + emptyFileIndex++; + file.Size = 0; + file.Crc = null; + } + } + } + finally { Log.PopIndent(); } + } + + #endregion + + #region Public Methods + + public void Open(Stream stream) + { + Close(); + + _streamOrigin = stream.Position; + _streamEnding = stream.Length; + + // TODO: Check Signature! + _header = new byte[0x20]; + for(int offset = 0; offset < 0x20; ) + { + int delta = stream.Read(_header, offset, 0x20 - offset); + if(delta == 0) + throw new EndOfStreamException(); + offset += delta; + } + + _stream = stream; + } + + public void Close() + { + if(_stream != null) + _stream.Dispose(); + + foreach(var stream in _cachedStreams.Values) + stream.Dispose(); + + _cachedStreams.Clear(); + } + + public ArchiveDatabase ReadDatabase(IPasswordProvider pass) + { + var db = new ArchiveDatabase(); + db.Clear(); + + db.MajorVersion = _header[6]; + db.MinorVersion = _header[7]; + + if(db.MajorVersion != 0) + throw new InvalidOperationException(); + + uint crcFromArchive = DataReader.Get32(_header, 8); + long nextHeaderOffset = (long)DataReader.Get64(_header, 0xC); + long nextHeaderSize = (long)DataReader.Get64(_header, 0x14); + uint nextHeaderCrc = DataReader.Get32(_header, 0x1C); + + uint crc = CRC.kInitCRC; + crc = CRC.Update(crc, nextHeaderOffset); + crc = CRC.Update(crc, nextHeaderSize); + crc = CRC.Update(crc, nextHeaderCrc); + crc = CRC.Finish(crc); + + if(crc != crcFromArchive) + throw new InvalidOperationException(); + + db.StartPositionAfterHeader = _streamOrigin + 0x20; + + // empty header is ok + if(nextHeaderSize == 0) + { + db.Fill(); + return db; + } + + + if(nextHeaderOffset < 0 || nextHeaderSize < 0 || nextHeaderSize > Int32.MaxValue) + throw new InvalidOperationException(); + + if(nextHeaderOffset > _streamEnding - db.StartPositionAfterHeader) + throw new IndexOutOfRangeException(); + + _stream.Seek(nextHeaderOffset, SeekOrigin.Current); + + byte[] header = new byte[nextHeaderSize]; + _stream.ReadExact(header, 0, header.Length); + + if(CRC.Finish(CRC.Update(CRC.kInitCRC, header, 0, header.Length)) != nextHeaderCrc) + throw new InvalidOperationException(); + + using(CStreamSwitch streamSwitch = new CStreamSwitch()) + { + streamSwitch.Set(this, header); + + BlockType? type = ReadId(); + if(type != BlockType.Header) + { + if(type != BlockType.EncodedHeader) + throw new InvalidOperationException(); + + var dataVector = ReadAndDecodePackedStreams(db.StartPositionAfterHeader, pass); + + // compressed header without content is odd but ok + if (dataVector.Count == 0) + { + db.Fill(); + return db; + } + + if(dataVector.Count != 1) + throw new InvalidOperationException(); + + streamSwitch.Set(this, dataVector[0]); + + if(ReadId() != BlockType.Header) + throw new InvalidOperationException(); + } + + ReadHeader(db, pass); + } + db.Fill(); + return db; + } + + internal class CExtractFolderInfo + { + internal int FileIndex; + internal int FolderIndex; + internal List ExtractStatuses = new List(); + internal CExtractFolderInfo(int fileIndex, int folderIndex) + { + FileIndex = fileIndex; + FolderIndex = folderIndex; + if(fileIndex != -1) + ExtractStatuses.Add(true); + } + } + + private class FolderUnpackStream: Stream + { + private ArchiveDatabase _db; + private int _otherIndex; + private int _startIndex; + private List _extractStatuses; + + public FolderUnpackStream(ArchiveDatabase db, int p, int startIndex, List list) + { + this._db = db; + this._otherIndex = p; + this._startIndex = startIndex; + this._extractStatuses = list; + } + + #region Stream + + public override bool CanRead + { + get { throw new NotImplementedException(); } + } + + public override bool CanSeek + { + get { throw new NotImplementedException(); } + } + + public override bool CanWrite + { + get { throw new NotImplementedException(); } + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override long Length + { + get { throw new NotImplementedException(); } + } + + public override long Position + { + get + { + throw new NotImplementedException(); + } + set + { + throw new NotImplementedException(); + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + private Stream _stream; + private long _rem; + private int _currentIndex; + private void ProcessEmptyFiles() + { + while(_currentIndex < _extractStatuses.Count && _db.Files[_startIndex + _currentIndex].Size == 0) + { + OpenFile(); + _stream.Dispose(); + _stream = null; + _currentIndex++; + } + } + private void OpenFile() + { + bool skip = !_extractStatuses[_currentIndex]; + int index = _startIndex + _currentIndex; + int realIndex = _otherIndex + index; + //string filename = @"D:\_testdump\" + _db.Files[index].Name; + //Directory.CreateDirectory(Path.GetDirectoryName(filename)); + //_stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Delete); + Log.WriteLine(_db.Files[index].Name); + if(_db.Files[index].CrcDefined) + _stream = new CrcCheckStream(_db.Files[index].Crc.Value); + else + _stream = new MemoryStream(); + _rem = _db.Files[index].Size; + } + public override void Write(byte[] buffer, int offset, int count) + { + while(count != 0) + { + if(_stream != null) + { + int write = count; + if(write > _rem) + write = (int)_rem; + _stream.Write(buffer, offset, write); + count -= write; + _rem -= write; + offset += write; + if(_rem == 0) + { + _stream.Dispose(); + _stream = null; + _currentIndex++; + ProcessEmptyFiles(); + } + } + else + { + ProcessEmptyFiles(); + if(_currentIndex == _extractStatuses.Count) + { + // we support partial extracting + System.Diagnostics.Debugger.Break(); + throw new NotImplementedException(); + } + OpenFile(); + } + } + } + + #endregion + } + + private Stream GetCachedDecoderStream(ArchiveDatabase _db, int folderIndex, IPasswordProvider pw) + { + Stream s; + if(!_cachedStreams.TryGetValue(folderIndex, out s)) + { + CFolder folderInfo = _db.Folders[folderIndex]; + int packStreamIndex = _db.Folders[folderIndex].FirstPackStreamId; + long folderStartPackPos = _db.GetFolderStreamPos(folderInfo, 0); + List packSizes = new List(); + for(int j = 0; j < folderInfo.PackStreams.Count; j++) + packSizes.Add(_db.PackSizes[packStreamIndex + j]); + + s = DecoderStreamHelper.CreateDecoderStream(_stream, folderStartPackPos, packSizes.ToArray(), folderInfo, pw); + _cachedStreams.Add(folderIndex, s); + } + return s; + } + + public Stream OpenStream(ArchiveDatabase _db, int fileIndex, IPasswordProvider pw) + { + int folderIndex = _db.FileIndexToFolderIndexMap[fileIndex]; + int numFilesInFolder = _db.NumUnpackStreamsVector[folderIndex]; + int firstFileIndex = _db.FolderStartFileIndex[folderIndex]; + if(firstFileIndex > fileIndex || fileIndex - firstFileIndex >= numFilesInFolder) + throw new InvalidOperationException(); + + int skipCount = fileIndex - firstFileIndex; + long skipSize = 0; + for(int i = 0; i < skipCount; i++) + skipSize += _db.Files[firstFileIndex + i].Size; + + Stream s = GetCachedDecoderStream(_db, folderIndex, pw); + s.Position = skipSize; + return new ReadOnlySubStream(s, _db.Files[fileIndex].Size); + } + + public void Extract(ArchiveDatabase _db, int[] indices, IPasswordProvider pw) + { + int numItems; + bool allFilesMode = (indices == null); + if(allFilesMode) + numItems = _db.Files.Count; + else + numItems = indices.Length; + + if(numItems == 0) + return; + + List extractFolderInfoVector = new List(); + for(int i = 0; i < numItems; i++) + { + int fileIndex = allFilesMode ? i : indices[i]; + + int folderIndex = _db.FileIndexToFolderIndexMap[fileIndex]; + if(folderIndex == -1) + { + extractFolderInfoVector.Add(new CExtractFolderInfo(fileIndex, -1)); + continue; + } + + if(extractFolderInfoVector.Count == 0 || folderIndex != extractFolderInfoVector.Last().FolderIndex) + extractFolderInfoVector.Add(new CExtractFolderInfo(-1, folderIndex)); + + CExtractFolderInfo efi = extractFolderInfoVector.Last(); + + int startIndex = _db.FolderStartFileIndex[folderIndex]; + for(int index = efi.ExtractStatuses.Count; index <= fileIndex - startIndex; index++) + efi.ExtractStatuses.Add(index == fileIndex - startIndex); + } + + foreach(CExtractFolderInfo efi in extractFolderInfoVector) + { + int startIndex; + if(efi.FileIndex != -1) + startIndex = efi.FileIndex; + else + startIndex = _db.FolderStartFileIndex[efi.FolderIndex]; + + var outStream = new FolderUnpackStream(_db, 0, startIndex, efi.ExtractStatuses); + + if(efi.FileIndex != -1) + continue; + + int folderIndex = efi.FolderIndex; + CFolder folderInfo = _db.Folders[folderIndex]; + + int packStreamIndex = _db.Folders[folderIndex].FirstPackStreamId; + long folderStartPackPos = _db.GetFolderStreamPos(folderInfo, 0); + + List packSizes = new List(); + for(int j = 0; j < folderInfo.PackStreams.Count; j++) + packSizes.Add(_db.PackSizes[packStreamIndex + j]); + + // TODO: If the decoding fails the last file may be extracted incompletely. Delete it? + + Stream s = DecoderStreamHelper.CreateDecoderStream(_stream, folderStartPackPos, packSizes.ToArray(), folderInfo, pw); + byte[] buffer = new byte[4 << 10]; + for(; ; ) + { + int processed = s.Read(buffer, 0, buffer.Length); + if(processed == 0) break; + outStream.Write(buffer, 0, processed); + } + } + } + + public IEnumerable GetFiles(ArchiveDatabase db) + { + return db.Files; + } + + public int GetFileIndex(ArchiveDatabase db, CFileItem item) + { + return db.Files.IndexOf(item); + } + + #endregion + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/ArchiveWriter.cs b/SharpCompress/Common/SevenZip/ArchiveWriter.cs new file mode 100644 index 00000000..52f76b6f --- /dev/null +++ b/SharpCompress/Common/SevenZip/ArchiveWriter.cs @@ -0,0 +1,1783 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +// TODO: Replace BinaryWriter by a custom one which is always little-endian. + +// Design: +// - constructor writes header for empty 7z archive +// - call methods to add content to the archive +// this will also build up header information in memory +// - call WriteFinalHeader to output the secondary header and update the primary header +// +// In any case the archive will be valid, but unless you call WriteFinalHeader +// it will just be an empty archive possibly followed by unreachable content. +// + +namespace ManagedLzma.LZMA.Master.SevenZip +{ + // TODO: Can we get rid of this interface and just pass the arguments directly to BeginWriteStream? + public interface IArchiveWriterEntry + { + string Name { get; } + FileAttributes? Attributes { get; } + DateTime? CreationTime { get; } + DateTime? LastWriteTime { get; } + DateTime? LastAccessTime { get; } + } + + internal static class FileNameHelper + { + public static string CalculateName(DirectoryInfo root, DirectoryInfo folder) + { + if(root.Root.FullName != folder.Root.FullName) + throw new InvalidOperationException("Unrelated directories."); + + Stack rootList = new Stack(); + while(root.FullName != root.Root.FullName) + { + rootList.Push(root); + root = root.Parent; + } + + Stack itemList = new Stack(); + while(folder.FullName != folder.Root.FullName) + { + itemList.Push(folder); + folder = folder.Parent; + } + + while(rootList.Count != 0 && itemList.Count != 0 && rootList.Peek().Name == itemList.Peek().Name) + { + rootList.Pop(); + itemList.Pop(); + } + + if(rootList.Count != 0) + throw new InvalidOperationException("Item is not contained in root."); + + if(itemList.Count == 0) + return null; + + return String.Join("/", itemList.Select(item => item.Name)); + } + + public static string CalculateName(DirectoryInfo root, FileInfo file) + { + string path = CalculateName(root, file.Directory); + return String.IsNullOrEmpty(path) ? file.Name : path + "/" + file.Name; + } + } + + internal sealed class FileBasedArchiveWriterEntry: IArchiveWriterEntry + { + private FileInfo mFile; + private string mName; + + public FileBasedArchiveWriterEntry(DirectoryInfo root, FileInfo file) + { + mFile = file; + mName = FileNameHelper.CalculateName(root, file); + } + + public string Name + { + get { return mName; } + } + + public FileAttributes? Attributes + { + get { return mFile.Attributes; } + } + + public DateTime? CreationTime + { + get { return mFile.CreationTimeUtc; } + } + + public DateTime? LastWriteTime + { + get { return mFile.LastWriteTimeUtc; } + } + + public DateTime? LastAccessTime + { + get { return mFile.LastAccessTimeUtc; } + } + } + + /// + /// MemoryStream which uses chunks of memory instead of one big byte array. + /// + internal sealed class FragmentedMemoryStream: Stream + { + #region Constants + + // Chunk size should be small enough for buffers to not be placed on + // the large object heap, so they stay relocable and don't fragment. + // Objects are placed on the LOH if they are larger than 85000 bytes, + // so 64K buffers should be fine for us. + + private const int kChunkShift = 16; + private const int kChunkSize = 1 << kChunkShift; + private const int kChunkMask = kChunkSize - 1; + + #endregion + + #region Variables + + private List mChunks = new List(); + private int mOffset; + private int mEnding; + + #endregion + + #region Public Methods + + public FragmentedMemoryStream() + { + } + + public FragmentedMemoryStream(int capacity) + { + EnsureCapacity(capacity); + } + + public int Capacity + { + get { return mChunks.Count << kChunkShift; } + } + + public void ReduceCapacity() + { + int requiredChunks = (mEnding + kChunkMask) >> kChunkShift; + + if(mChunks.Count > requiredChunks) + mChunks.RemoveRange(requiredChunks, mChunks.Count - requiredChunks); + } + + public void ReduceCapacity(int capacity) + { + if(capacity < 0) + throw new ArgumentOutOfRangeException("capacity"); + + // Limit stream size to prevent overflows in calculations. + if(capacity > Int32.MaxValue - kChunkMask) + throw new NotSupportedException("Large streams are not supported."); + + int requiredChunks = (capacity + kChunkMask) >> kChunkShift; + + if(mChunks.Count > requiredChunks) + mChunks.RemoveRange(requiredChunks, mChunks.Count - requiredChunks); + } + + public void EnsureCapacity(int capacity) + { + if(capacity < 0) + throw new ArgumentOutOfRangeException("capacity"); + + // Limit stream size to prevent overflows in calculations. + if(capacity > Int32.MaxValue - kChunkMask) + throw new NotSupportedException("Large streams are not supported."); + + int requiredChunks = (capacity + kChunkMask) >> kChunkShift; + + while(mChunks.Count < requiredChunks) + mChunks.Add(new byte[kChunkSize]); + } + + #endregion + + public void FullCopyTo(Stream destination) + { + int fullCount = mEnding >> kChunkShift; + for(int i = 0; i < fullCount; i++) + destination.Write(mChunks[i], 0, kChunkSize); + + int remaining = mEnding & kChunkMask; + if(remaining != 0) + destination.Write(mChunks[fullCount], 0, remaining); + } + + #region Stream Implementation + + protected override void Dispose(bool disposing) + { + if(disposing) + mChunks = null; + + base.Dispose(disposing); + } + + public override bool CanSeek + { + get { return true; } + } + + public override long Position + { + get { return mOffset; } + set + { + if(value < 0 || value > mEnding) + throw new ArgumentOutOfRangeException("value"); + + mOffset = (int)value; + } + } + + public override long Length + { + get { return mEnding; } + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch(origin) + { + case SeekOrigin.Begin: + return Position = offset; + case SeekOrigin.Current: + return Position += offset; + case SeekOrigin.End: + return Position = Length + offset; + default: + throw new ArgumentOutOfRangeException("origin"); + } + } + + public override bool CanRead + { + get { return true; } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if(buffer == null) + throw new ArgumentNullException("buffer"); + + if(offset < 0 || offset > buffer.Length) + throw new ArgumentOutOfRangeException("offset"); + + if(count < 0 || count > buffer.Length - offset) + throw new ArgumentOutOfRangeException("count"); + + int remaining = mEnding - mOffset; + if(count > remaining) + count = remaining; + + if(count == 0) + return 0; + + byte[] chunk = mChunks[mOffset >> kChunkShift]; + int chunkOffset = mOffset & kChunkMask; + int readLength = Math.Min(kChunkSize - chunkOffset, count); + mOffset += readLength; + Buffer.BlockCopy(chunk, chunkOffset, buffer, offset, readLength); + return readLength; + } + + public override int ReadByte() + { + if(mOffset == mEnding) + return -1; + + byte value = mChunks[mOffset >> kChunkShift][mOffset & kChunkMask]; + mOffset++; + return value; + } + + public override bool CanWrite + { + get { return true; } + } + + public override void Flush() { } + + public override void Write(byte[] buffer, int offset, int count) + { + if(buffer == null) + throw new ArgumentNullException("buffer"); + + if(offset < 0 || offset > buffer.Length) + throw new ArgumentOutOfRangeException("offset"); + + if(count < 0 || count > buffer.Length - offset) + throw new ArgumentOutOfRangeException("count"); + + int remaining = mEnding - mOffset; + if(count > remaining) + { + SetLength((long)mOffset + (long)count); + remaining = mEnding - mOffset; + } + + while(count > 0) + { + byte[] chunk = mChunks[mOffset >> kChunkShift]; + int chunkOffset = mOffset & kChunkMask; + int writeLength = Math.Min(kChunkSize - chunkOffset, count); + mOffset += writeLength; + Buffer.BlockCopy(buffer, offset, chunk, chunkOffset, writeLength); + offset += writeLength; + count -= writeLength; + } + } + + public override void WriteByte(byte value) + { + if(mOffset == mEnding) + SetLength(mEnding + 1); + + mChunks[mOffset >> kChunkShift][mOffset & kChunkMask] = value; + mOffset++; + } + + public override void SetLength(long value) + { + if(value < 0) + throw new ArgumentOutOfRangeException("value"); + + // Limit stream size to prevent overflows in calculations. + if(value > Int32.MaxValue - kChunkMask) + throw new NotSupportedException("Large streams are not supported."); + + mEnding = (int)value; + + if(mOffset > mEnding) + mOffset = mEnding; + + int requiredChunks = (mEnding + kChunkMask) >> kChunkShift; + + // TODO: figure out a reasonable strategy about how many chunks to retain + + //if(mChunks.Count >= requiredChunks) + //{ + // // keep one chunk more than required, in case we start writing again + // if(mChunks.Count > ++requiredChunks) + // mChunks.RemoveRange(requiredChunks, mChunks.Count - requiredChunks); + // return; + //} + + while(mChunks.Count < requiredChunks) + mChunks.Add(new byte[kChunkSize]); + } + + #endregion + } + + public class ArchiveWriter + { + #region Configuration Elements + + internal abstract class StreamRef + { + public abstract long GetSize(FileSet fileset); + public abstract uint? GetHash(FileSet fileset); + } + + internal class InputStreamRef: StreamRef + { + public int PackedStreamIndex; + + public override long GetSize(FileSet fileset) + { + return fileset.InputStreams[PackedStreamIndex].Size; + } + + public override uint? GetHash(FileSet fileset) + { + return fileset.InputStreams[PackedStreamIndex].Hash; + } + } + + internal class CoderStreamRef: StreamRef + { + public int CoderIndex; + public int StreamIndex; + + public override long GetSize(FileSet fileset) + { + return fileset.Coders[CoderIndex].OutputStreams[StreamIndex].Size; + } + + public override uint? GetHash(FileSet fileset) + { + return fileset.Coders[CoderIndex].OutputStreams[StreamIndex].Hash; + } + } + + internal class Coder + { + public master._7zip.Legacy.CMethodId MethodId; + public byte[] Settings; + public StreamRef[] InputStreams; + public CoderStream[] OutputStreams; + } + + internal class CoderStream + { + public long Size; + public uint? Hash; + } + + internal class InputStream + { + public long Size; + public uint? Hash; + } + + internal class FileSet + { + public InputStream[] InputStreams; + public Coder[] Coders; + public StreamRef DataStream; + public FileEntry[] Files; + } + + internal class FileEntry + { + public string Name; + public uint? Flags; + public DateTime? CTime; + public DateTime? MTime; + public DateTime? ATime; + public long Size; + public uint Hash; + } + + #endregion + + #region Encoders + + internal sealed class EncoderStream: Stream + { + private long mStreamSize; + private Encoder mEncoder; + private uint mCRC = CRC.kInitCRC; + + internal EncoderStream(Encoder encoder) + { + mEncoder = encoder; + } + + protected override void Dispose(bool disposing) + { + if(disposing && mEncoder != null) + { + mEncoder = null; + mCRC = ~mCRC; + } + + base.Dispose(disposing); + } + + public sealed override bool CanRead + { + get { return false; } + } + + public sealed override bool CanSeek + { + get { return false; } + } + + public sealed override bool CanWrite + { + get { return true; } + } + + public sealed override long Length + { + get { throw new NotSupportedException(); } + } + + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public sealed override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public sealed override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public sealed override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Flush() { } + + public override void Write(byte[] buffer, int offset, int count) + { + if(buffer == null) + throw new ArgumentNullException("buffer"); + + if(offset < 0 || offset > buffer.Length) + throw new ArgumentOutOfRangeException("offset"); + + if(count < 0 || count > buffer.Length - offset) + throw new ArgumentOutOfRangeException("count"); + + mStreamSize += count; + + mEncoder.WriteInput(buffer, offset, count); + } + + internal void Close(FileEntry file) + { + Close(); + + file.Hash = mCRC; + file.Size = mStreamSize; + } + } + + internal sealed class BufferedFileSet + { + private FileSet mMetadata; + private FragmentedMemoryStream mBuffer; + + public BufferedFileSet(FileSet metadata, FragmentedMemoryStream buffer) + { + mMetadata = metadata; + mBuffer = buffer; + } + + public FileSet Metadata + { + get { return mMetadata; } + } + + public FragmentedMemoryStream Buffer + { + get { return mBuffer; } + } + } + + public abstract class Encoder: IDisposable + { + private static DateTime? EnsureUTC(DateTime? value) + { + if(value.HasValue) + return value.Value.ToUniversalTime(); + else + return null; + } + + private long mInputSize; + private long mOutputSize; + private ArchiveWriter mWriter; + private FragmentedMemoryStream mBuffer; + private EncoderStream mCurrentStream; + private List mFiles; + private FileEntry mCurrentFile; + private List mFlushed; + + public abstract long LowerBound { get; } + public abstract long UpperBound { get; } + + internal Encoder() { } + + public virtual void Dispose() + { + Disconnect(); + } + + internal void Connect(ArchiveWriter writer) + { + Debug.Assert(writer != null && mWriter == null); + + mWriter = writer; + + if(mFlushed != null && mFlushed.Count != 0) + { + foreach(var item in mFlushed) + mWriter.Encoder_WriteFileSet(item); + + mFlushed.Clear(); + } + + if(mBuffer != null) + { + mBuffer.FullCopyTo(writer.mFileStream); + mBuffer.SetLength(0); + } + } + + public void Disconnect() + { + if(IsConnected) + { + Flush(); + + Debug.Assert(mWriter.mEncoder == this); + mWriter.mEncoder = null; + mWriter = null; + } + } + + private void FinishCurrentFile() + { + if(mCurrentFile != null) + { + mCurrentStream.Close(mCurrentFile); + mCurrentStream = null; + + if(mFiles == null) + mFiles = new List(); + + mFiles.Add(mCurrentFile); + mCurrentFile = null; + } + } + + public ArchiveWriter ArchiveWriter + { + get { return mWriter; } + } + + public bool IsConnected + { + get { return mWriter != null; } + } + + public Stream BeginWriteFile(IArchiveWriterEntry file) + { + FinishCurrentFile(); + + if(file == null) + throw new ArgumentNullException("file"); + + mCurrentFile = new FileEntry { + Name = file.Name, + CTime = EnsureUTC(file.CreationTime), + MTime = EnsureUTC(file.LastWriteTime), + ATime = EnsureUTC(file.LastAccessTime), + }; + + var attributes = file.Attributes; + if(attributes.HasValue) + mCurrentFile.Flags = (uint)attributes.Value; + + if(mWriter == null && mBuffer == null) + mBuffer = new FragmentedMemoryStream(); + + return mCurrentStream = new EncoderStream(this); + } + + /// + /// Closes the currently opened file stream and flushes buffered metadata. + /// + public void Flush() + { + FinishCurrentFile(); + + if(mFiles != null && mFiles.Count != 0) + { + PrepareFinishFileSet(); + var fileset = FinishFileSet(mFiles.ToArray(), mInputSize, mOutputSize); + mFiles.Clear(); + mInputSize = 0; + mOutputSize = 0; + + if(IsConnected) + { + mWriter.Encoder_FinishFileSet(fileset); + } + else + { + if(mFlushed == null) + mFlushed = new List(); + + Debug.Assert(mBuffer != null); + mFlushed.Add(new BufferedFileSet(fileset, mBuffer)); + mBuffer = null; + } + } + } + + /// + /// Write flushed data to an ArchiveWriter without connecting to it. + /// If the ArchiveWriter has a connected encoder it is flushed first. + /// This does not flush this encoder, if that is required do it manually. + /// + public void WriteFlushedData(ArchiveWriter writer) + { + if(writer == null) + throw new ArgumentNullException("writer"); + + // If the writer has a connected encoder we need to flush that one. + if(writer.mEncoder != null) + writer.mEncoder.Flush(); + + if(mFlushed != null && mFlushed.Count != 0) + { + foreach(var item in mFlushed) + writer.Encoder_WriteFileSet(item); + + mFlushed.Clear(); + } + } + + internal virtual void PrepareFinishFileSet() { } + internal abstract FileSet FinishFileSet(FileEntry[] entries, long inputSize, long outputSize); + internal abstract void OnWriteInput(byte[] buffer, int offset, int length); + + internal void WriteInput(byte[] buffer, int offset, int length) + { + OnWriteInput(buffer, offset, length); + + // If we didn't throw we assume the input was processed. + mInputSize += length; + } + + #region Protected Methods + + protected long CurrentInputSize + { + get { return mInputSize; } + } + + protected long CurrentOutputSize + { + get { return mOutputSize; } + } + + protected void WriteOutput(byte[] buffer, int offset, int length) + { + if(mWriter != null) + mWriter.mFileStream.Write(buffer, offset, length); + else + mBuffer.Write(buffer, offset, length); + + // If we didn't throw we assume the output was processed. + mOutputSize += length; + } + + #endregion + } + + public sealed class PlainEncoder: Encoder + { + public PlainEncoder() { } + + public override long LowerBound + { + get { return CurrentOutputSize; } + } + + public override long UpperBound + { + get { return CurrentOutputSize; } + } + + internal override void OnWriteInput(byte[] buffer, int offset, int length) + { + WriteOutput(buffer, offset, length); + } + + internal override FileSet FinishFileSet(FileEntry[] entries, long inputSize, long outputSize) + { + Debug.Assert(inputSize == outputSize); + + return new FileSet { + Files = entries, + DataStream = new InputStreamRef { PackedStreamIndex = 0 }, + InputStreams = new[] { new InputStream { Size = outputSize } }, + Coders = new[] { new Coder { + MethodId = master._7zip.Legacy.CMethodId.kCopy, + Settings = null, + InputStreams = new[] { new InputStreamRef { PackedStreamIndex = 0 } }, + OutputStreams = new[] { new CoderStream { Size = inputSize } }, + } }, + }; + } + } + + public abstract class ThreadedEncoder: Encoder + { + private enum State + { + Ready = 0, + Flush = 1, + Dispose = 2, + } + + private const int kBufferLength = 1 << 20; + + private object mSyncObject; + private Thread mEncoderThread; + private State mState; + private byte[] mInputBuffer; + private int mInputOffset; + private int mInputEnding; + private int mBufferOffset; // not locked (not accessed by thread) + private int mBufferEnding; // not locked (not accessed by thread) + + // HACK: To allow the user to react when the buffering crosses a certain threshold. Need a better API for this. + private int mOutputThreshold; + private bool mTriggerOutputThreshold; + public event EventHandler OnOutputThresholdReached; + public void SetOutputThreshold(int threshold) + { + lock(mSyncObject) + { + mTriggerOutputThreshold = false; + mOutputThreshold = threshold; + } + } + + internal ThreadedEncoder() + { + mSyncObject = new object(); + mInputBuffer = new byte[kBufferLength]; + mBufferEnding = kBufferLength - 1; + } + + protected void StartThread(string threadName) + { + mEncoderThread = new Thread(EncoderThread); + mEncoderThread.Name = threadName; + mEncoderThread.Start(); + } + + public override void Dispose() + { + try + { + base.Dispose(); + } + finally + { + lock(mSyncObject) + { + if(mState == State.Dispose) + goto skip; + + // If we are in flushed state the owner thread should be blocking inside FinishFileSet. + // So getting here in flushed state means some foreign thread called us, which is invalid. + Utils.Assert(mState == State.Ready); + mState = State.Dispose; + Monitor.Pulse(mSyncObject); + } + + mEncoderThread.Join(); + + skip: { } + } + } + + internal sealed override void OnWriteInput(byte[] buffer, int offset, int count) + { + if(mInputBuffer == null) + throw new ObjectDisposedException(null); + + // HACK: this is really ugly and probably hurts performance, but whatever, don't have a better solution currently + // it must be done at this place because at the place where we calculate the threshold we are on the wrong thread + bool trigger = false; + lock(mSyncObject) + { + if(mTriggerOutputThreshold) + { + mTriggerOutputThreshold = false; + trigger = true; + System.Diagnostics.Debug.WriteLine("output threshold triggered"); + } + } + if(trigger) + { + var handler = OnOutputThresholdReached; + if(handler != null) + handler(this, EventArgs.Empty); + } + // HACK END + + while(count > 0) + { + int copy; + if(mBufferOffset <= mBufferEnding) + copy = Math.Min(count, mBufferEnding - mBufferOffset); + else + copy = Math.Min(count, kBufferLength - mBufferOffset); + + if(copy > 0) + { + Buffer.BlockCopy(buffer, offset, mInputBuffer, mBufferOffset, copy); + count -= copy; + offset += copy; + mBufferOffset = (mBufferOffset + copy) % kBufferLength; + } + + lock(mSyncObject) + { + if(copy > 0) + { + mInputEnding = mBufferOffset; + Monitor.Pulse(mSyncObject); + } + + for(; ; ) + { + if(mState != State.Ready) + throw new ObjectDisposedException(null); + + int offsetMinusOne = (mInputOffset + kBufferLength - 1) % kBufferLength; + if(mInputEnding != offsetMinusOne) + { + mBufferEnding = offsetMinusOne; + break; + } + + Monitor.Wait(mSyncObject); + } + } + } + } + + internal override void PrepareFinishFileSet() + { + lock(mSyncObject) + { + Utils.Assert(mState == State.Ready); + + while(mInputOffset != mInputEnding) + Monitor.Wait(mSyncObject); + + mState = State.Flush; + Monitor.Pulse(mSyncObject); + + do { Monitor.Wait(mSyncObject); } + while(mState == State.Flush); + } + } + + private void EncoderThread() + { + for(; ; ) + { + EncoderThreadLoop(); + + lock(mSyncObject) + { + if(mState == State.Dispose) + return; + + Utils.Assert(mState == State.Flush); + mState = State.Ready; + Monitor.Pulse(mSyncObject); + } + } + } + + protected abstract void EncoderThreadLoop(); + + protected int ReadInputAsync(byte[] buffer, int offset, int length) + { + lock(mSyncObject) + { + for(; ; ) + { + if(mState != State.Ready) + return 0; + + if(mInputOffset != mInputEnding) + { + int size; + if(mInputOffset <= mInputEnding) + size = Math.Min(length, mInputEnding - mInputOffset); + else + size = Math.Min(length, kBufferLength - mInputOffset); + + Utils.Assert(size != 0); + Buffer.BlockCopy(mInputBuffer, mInputOffset, buffer, offset, size); + mInputOffset = (mInputOffset + size) % kBufferLength; + Monitor.Pulse(mSyncObject); + return size; + } + + Monitor.Wait(mSyncObject); + } + } + } + + protected void WriteOutputAsync(byte[] buffer, int offset, int length) + { + // TODO: is it sane to write without lock here? + // need to check both for buffered and connected output cases + // -> it probably is not sane because the base class increments that CurrentOutputSize counter + // which is also read from the main thread to estimate bounds -> need to fix that counter issue + WriteOutput(buffer, offset, length); + + // HACK: this doesn't really belong here, but I don't have any idea how to provide equivalent functionality + // maybe let the caller provide the buffer stream so the caller can hook into it through subclassing + // (note that we are on the wrong thread and can't call the event handler directly) + lock(mSyncObject) + { + if(mOutputThreshold > 0) + { + mOutputThreshold -= length; + if(mOutputThreshold <= 0) + { + mTriggerOutputThreshold = true; + System.Diagnostics.Debug.WriteLine("output threshold reached"); + } + } + } + } + } + + public sealed class LzmaEncoder: ThreadedEncoder + { + private long mLowerBound; + private object mSyncObject; + private LZMA.CSeqOutStream mOutputHelper; + private LZMA.CSeqInStream mInputHelper; + private byte[] mSettings; + + public LzmaEncoder() + { + mSyncObject = new object(); + mOutputHelper = new LZMA.CSeqOutStream(WriteOutputHelper); + mInputHelper = new LZMA.CSeqInStream(ReadInputHelper); + StartThread("LZMA Stream Buffer Thread"); + } + + public override long LowerBound + { + get + { + lock(mSyncObject) + return mLowerBound; + } + } + + public override long UpperBound + { + get { return CurrentInputSize; } + } + + private long ReadInputHelper(P buf, long sz) + { + Utils.Assert(sz != 0); + return ReadInputAsync(buf.mBuffer, buf.mOffset, checked((int)sz)); + } + + private void WriteOutputHelper(P buf, long sz) + { + WriteOutputAsync(buf.mBuffer, buf.mOffset, checked((int)sz)); + lock(mSyncObject) + mLowerBound += sz; + } + + protected override void EncoderThreadLoop() + { + var settings = LZMA.CLzmaEncProps.LzmaEncProps_Init(); + var encoder = LZMA.LzmaEnc_Create(LZMA.ISzAlloc.SmallAlloc); + var res = encoder.LzmaEnc_SetProps(settings); + if(res != LZMA.SZ_OK) + throw new InvalidOperationException(); + + mSettings = new byte[LZMA.LZMA_PROPS_SIZE]; + long binarySettingsSize = LZMA.LZMA_PROPS_SIZE; + res = encoder.LzmaEnc_WriteProperties(mSettings, ref binarySettingsSize); + if(res != LZMA.SZ_OK) + throw new InvalidOperationException(); + if(binarySettingsSize != LZMA.LZMA_PROPS_SIZE) + throw new NotSupportedException(); + + res = encoder.LzmaEnc_Encode(mOutputHelper, mInputHelper, null, LZMA.ISzAlloc.SmallAlloc, LZMA.ISzAlloc.BigAlloc); + if(res != LZMA.SZ_OK) + throw new InvalidOperationException(); + + encoder.LzmaEnc_Destroy(LZMA.ISzAlloc.SmallAlloc, LZMA.ISzAlloc.BigAlloc); + } + + internal override FileSet FinishFileSet(FileEntry[] entries, long inputSize, long outputSize) + { + return new FileSet { + Files = entries, + DataStream = new CoderStreamRef { CoderIndex = 0, StreamIndex = 0 }, + InputStreams = new[] { new InputStream { Size = outputSize } }, + Coders = new[] { new Coder { + MethodId = master._7zip.Legacy.CMethodId.kLzma, + Settings = mSettings, + InputStreams = new[] { new InputStreamRef { PackedStreamIndex = 0 } }, + OutputStreams = new[] { new CoderStream { Size = inputSize } }, + } }, + }; + } + } + + public sealed class Lzma2Encoder: ThreadedEncoder + { + private long mLowerBound; + private object mSyncObject; + private LZMA.CSeqOutStream mOutputHelper; + private LZMA.CSeqInStream mInputHelper; + private int? mThreadCount; + private byte mSettings; + + public Lzma2Encoder(int? threadCount) + { + mThreadCount = threadCount; + mSyncObject = new object(); + mOutputHelper = new LZMA.CSeqOutStream(WriteOutputHelper); + mInputHelper = new LZMA.CSeqInStream(ReadInputHelper); + StartThread("LZMA 2 Stream Buffer Thread"); + } + + public override long LowerBound + { + get + { + lock(mSyncObject) + return mLowerBound; + } + } + + public override long UpperBound + { + get { return CurrentInputSize; } + } + + private long ReadInputHelper(P buf, long sz) + { + Utils.Assert(sz != 0); + return ReadInputAsync(buf.mBuffer, buf.mOffset, checked((int)sz)); + } + + private void WriteOutputHelper(P buf, long sz) + { + WriteOutputAsync(buf.mBuffer, buf.mOffset, checked((int)sz)); + lock(mSyncObject) + mLowerBound += sz; + } + + protected override void EncoderThreadLoop() + { + var settings = new LZMA.CLzma2EncProps(); + settings.Lzma2EncProps_Init(); + + if(mThreadCount.HasValue) + settings.mNumBlockThreads = mThreadCount.Value; + + var encoder = new LZMA.CLzma2Enc(LZMA.ISzAlloc.SmallAlloc, LZMA.ISzAlloc.BigAlloc); + var res = encoder.Lzma2Enc_SetProps(settings); + if(res != LZMA.SZ_OK) + throw new InvalidOperationException(); + + mSettings = encoder.Lzma2Enc_WriteProperties(); + + res = encoder.Lzma2Enc_Encode(mOutputHelper, mInputHelper, null); + if(res != LZMA.SZ_OK) + throw new InvalidOperationException(); + + encoder.Lzma2Enc_Destroy(); + } + + internal override FileSet FinishFileSet(FileEntry[] entries, long inputSize, long outputSize) + { + return new FileSet { + Files = entries, + DataStream = new CoderStreamRef { CoderIndex = 0, StreamIndex = 0 }, + InputStreams = new[] { new InputStream { Size = outputSize } }, + Coders = new[] { new Coder { + MethodId = master._7zip.Legacy.CMethodId.kLzma2, + Settings = new byte[] { mSettings }, + InputStreams = new[] { new InputStreamRef { PackedStreamIndex = 0 } }, + OutputStreams = new[] { new CoderStream { Size = inputSize } }, + } }, + }; + } + } + + #endregion + + #region Constants & Variables + + private static readonly byte[] kSignature = { (byte)'7', (byte)'z', 0xBC, 0xAF, 0x27, 0x1C }; + + private long mFileOrigin; + private long mWrittenSync; + private Stream mFileStream; + private List mFileSets; + private Encoder mEncoder; // TODO: rename to mEncoder + + #endregion + + #region Public Methods + + public ArchiveWriter(Stream stream) + { + if(stream == null) + throw new ArgumentNullException("stream"); + + if(!stream.CanWrite) + throw new ArgumentException("Stream must support writing.", "stream"); + + // Seeking is required only because we need to go back to the start of the stream + // and fix up a reference to the header data. Possible solution: If we fix filesize + // ahead of time we could reserve space for the header and output a fixed offset. + if(!stream.CanSeek) + throw new ArgumentException("Stream must support seeking.", "stream"); + + // TODO: Implement the encoding independant of BinaryWriter endianess. + if(!BitConverter.IsLittleEndian) + throw new NotSupportedException("BinaryWriter must be little endian."); + + mFileStream = stream; + mFileOrigin = stream.Position; + + var writer = new BinaryWriter(stream, Encoding.Unicode); + + writer.Write(kSignature); + writer.Write((byte)0); + writer.Write((byte)3); + + // We don't have a header yet so just write a placeholder. As a side effect + // the placeholder will make this file look like a valid but empty archive. + WriteHeaderInfo(writer, 0, 0, CRC.Finish(CRC.kInitCRC)); + + mFileSets = new List(); + mWrittenSync = mFileStream.Position; + } + + /// + /// Returns the amount of data written so far. This is a lower bound for the + /// current archive size if it would be closed now. + /// + public long WrittenSize + { + get + { + // NOTE: If we have an encoder we shouldn't access the file stream because it's owned by the encoder. + + long size = mWrittenSync; + + if(mEncoder != null) + size += mEncoder.LowerBound; + + return size; + } + } + + /// + /// Returns an estimated limit for the file size if the archive would be closed now. + /// This consists of WrittenSize plus an estimated size limit for buffered data and the header. + /// The actual archive size may be smaller due to overestimating the header size. + /// + public long CurrentSizeLimit + { + get + { + // TODO: CalculateHeaderLimit does not include the metadata from the active encoder! + long size = mWrittenSync + CalculateHeaderLimit(); + + if(mEncoder != null) + size += mEncoder.UpperBound; + + return size; + } + } + + public void WriteFinalHeader() + { + if(mEncoder != null) + mEncoder.Flush(); + + var files = mFileSets.SelectMany(stream => stream.Files).ToArray(); + if(files.Length != 0) + { + long headerOffset = mFileStream.Position; + + var headerStream = new master._7zip.Legacy.CrcBuilderStream(mFileStream); + var writer = new BinaryWriter(headerStream, Encoding.Unicode); + + WriteNumber(writer, BlockType.Header); + + var inputStreams = mFileSets.SelectMany(fileset => fileset.InputStreams).ToArray(); + if(inputStreams.Any(stream => stream.Size != 0)) + { + WriteNumber(writer, BlockType.MainStreamsInfo); + WriteNumber(writer, BlockType.PackInfo); + WriteNumber(writer, (ulong)0); // offset to input streams + WriteNumber(writer, inputStreams.Length); + WriteNumber(writer, BlockType.Size); + foreach(var stream in inputStreams) + WriteNumber(writer, stream.Size); + WriteNumber(writer, BlockType.End); + WriteNumber(writer, BlockType.UnpackInfo); + WriteNumber(writer, BlockType.Folder); + WriteNumber(writer, mFileSets.Count); + writer.Write((byte)0); // inline data + foreach(var fileset in mFileSets) + { + WriteNumber(writer, fileset.Coders.Length); + foreach(var coder in fileset.Coders) + { + int idlen = coder.MethodId.GetLength(); + if(idlen >= 8) + throw new NotSupportedException(); + + int flags = idlen; + + if(coder.InputStreams.Length != 1 || coder.OutputStreams.Length != 1) + flags |= 0x10; + + if(coder.Settings != null) + flags |= 0x20; + + writer.Write((byte)flags); + + ulong id = coder.MethodId.Id; + for(int i = idlen - 1; i >= 0; i--) + writer.Write((byte)(id >> (i * 8))); + + if((flags & 0x10) != 0) + { + WriteNumber(writer, coder.InputStreams.Length); + WriteNumber(writer, coder.OutputStreams.Length); + } + + if((flags & 0x20) != 0) + { + WriteNumber(writer, coder.Settings.Length); + writer.Write(coder.Settings); + } + + // TODO: Bind pairs and association to streams ... + if(fileset.Coders.Length > 1 || coder.InputStreams.Length != 1 || coder.OutputStreams.Length != 1) + throw new NotSupportedException(); + } + } + WriteNumber(writer, BlockType.CodersUnpackSize); + foreach(var fileset in mFileSets) + WriteNumber(writer, fileset.DataStream.GetSize(fileset)); + WriteNumber(writer, BlockType.End); + WriteNumber(writer, BlockType.SubStreamsInfo); + WriteNumber(writer, BlockType.NumUnpackStream); + foreach(var stream in mFileSets) + WriteNumber(writer, stream.Files.Length); + WriteNumber(writer, BlockType.Size); + foreach(var stream in mFileSets) + for(int i = 0; i < stream.Files.Length - 1; i++) + WriteNumber(writer, stream.Files[i].Size); + WriteNumber(writer, BlockType.End); + WriteNumber(writer, BlockType.End); + } + + WriteNumber(writer, BlockType.FilesInfo); + WriteNumber(writer, files.Length); + + WriteNumber(writer, BlockType.Name); + WriteNumber(writer, 1 + files.Sum(file => file.Name.Length + 1) * 2); + writer.Write((byte)0); // inline names + for(int i = 0; i < files.Length; i++) + { + string name = files[i].Name; + for(int j = 0; j < name.Length; j++) + writer.Write(name[j]); + writer.Write('\0'); + } + + /* had to disable empty streams and files because above BlockType.Size doesn't respect them + * if a file is marked as empty stream it doesn't get a size/hash entry in the coder header above + * however, to fix that, we'd need to skip coders with only empty files too, so its easier to do it this way for now + if(files.Any(file => file.Size == 0)) + { + int emptyStreams = 0; + + WriteNumber(writer, BlockType.EmptyStream); + WriteNumber(writer, (files.Length + 7) / 8); + for(int i = 0; i < files.Length; i += 8) + { + int mask = 0; + for(int j = 0; j < 8; j++) + { + if(i + j < files.Length && files[i + j].Size == 0) + { + mask |= 1 << (7 - j); + emptyStreams++; + } + } + writer.Write((byte)mask); + } + + WriteNumber(writer, BlockType.EmptyFile); + WriteNumber(writer, (emptyStreams + 7) / 8); + for(int i = 0; i < emptyStreams; i += 8) + { + int mask = 0; + for(int j = 0; j < 8; j++) + if(i + j < emptyStreams) + mask |= 1 << (7 - j); + writer.Write((byte)mask); + } + } + */ + + int ctimeCount = files.Count(file => file.CTime.HasValue); + if(ctimeCount != 0) + { + WriteNumber(writer, BlockType.CTime); + + if(ctimeCount == files.Length) + { + WriteNumber(writer, 2 + ctimeCount * 8); + writer.Write((byte)1); + } + else + { + WriteNumber(writer, (ctimeCount + 7) / 8 + 2 + ctimeCount * 8); + writer.Write((byte)0); + + for(int i = 0; i < files.Length; i += 8) + { + int mask = 0; + for(int j = 0; j < 8; j++) + if(i + j < files.Length && files[i + j].CTime.HasValue) + mask |= 1 << (7 - j); + + writer.Write((byte)mask); + } + } + + writer.Write((byte)0); // inline data + + for(int i = 0; i < files.Length; i++) + if(files[i].CTime.HasValue) + writer.Write(files[i].CTime.Value.ToFileTimeUtc()); + } + + int atimeCount = files.Count(file => file.ATime.HasValue); + if(atimeCount != 0) + { + WriteNumber(writer, BlockType.ATime); + + if(atimeCount == files.Length) + { + WriteNumber(writer, 2 + atimeCount * 8); + writer.Write((byte)1); + } + else + { + WriteNumber(writer, (atimeCount + 7) / 8 + 2 + atimeCount * 8); + writer.Write((byte)0); + + for(int i = 0; i < files.Length; i += 8) + { + int mask = 0; + for(int j = 0; j < 8; j++) + if(i + j < files.Length && files[i + j].ATime.HasValue) + mask |= 1 << (7 - j); + + writer.Write((byte)mask); + } + } + + writer.Write((byte)0); // inline data + + for(int i = 0; i < files.Length; i++) + if(files[i].ATime.HasValue) + writer.Write(files[i].ATime.Value.ToFileTimeUtc()); + } + + int mtimeCount = files.Count(file => file.MTime.HasValue); + if(mtimeCount != 0) + { + WriteNumber(writer, BlockType.MTime); + + if(mtimeCount == files.Length) + { + WriteNumber(writer, 2 + mtimeCount * 8); + writer.Write((byte)1); + } + else + { + WriteNumber(writer, (mtimeCount + 7) / 8 + 2 + mtimeCount * 8); + writer.Write((byte)0); + + for(int i = 0; i < files.Length; i += 8) + { + int mask = 0; + for(int j = 0; j < 8; j++) + if(i + j < files.Length && files[i + j].MTime.HasValue) + mask |= 1 << (7 - j); + + writer.Write((byte)mask); + } + } + + writer.Write((byte)0); // inline data + + for(int i = 0; i < files.Length; i++) + if(files[i].MTime.HasValue) + writer.Write(files[i].MTime.Value.ToFileTimeUtc()); + } + + WriteNumber(writer, BlockType.End); + + uint headerCRC = headerStream.Finish(); + long headerSize = mFileStream.Position - headerOffset; + mFileStream.Position = mFileOrigin + 8; + WriteHeaderInfo(new BinaryWriter(mFileStream, Encoding.Unicode), headerOffset - mFileOrigin - 0x20, headerSize, headerCRC); + } + + mFileStream.Close(); // so we don't start overwriting stuff accidently by calling more functions + } + + #endregion + + #region Internal Methods + + // TODO: move into Encoder class? + internal void Encoder_FinishFileSet(FileSet fileset) + { + mFileSets.Add(fileset); + mWrittenSync = mFileStream.Position; + } + + // TODO: move into Encoder class? + internal void Encoder_WriteFileSet(BufferedFileSet fileset) + { + fileset.Buffer.FullCopyTo(mFileStream); + Encoder_FinishFileSet(fileset.Metadata); + } + + #endregion + + #region Encoding Methods + + /// + /// Returns the currently connected encoder. + /// + public Encoder CurrentEncoder + { + get { return mEncoder; } + } + + /// + /// Flushes the currently connected encoder and disconnects it. + /// + public void DisconnectEncoder() + { + if(mEncoder != null) + { + mEncoder.Disconnect(); + + Debug.Assert(mEncoder == null); + } + } + + /// + /// Flushes the currently connected encoder and selects the given encoder as current. + /// The connected encoder can write directly to the archive file with reduced buffering. + /// + public void ConnectEncoder(Encoder encoder) + { + if(encoder == null) + throw new ArgumentNullException("encoder"); + + if(mEncoder == encoder) + { + mEncoder.Flush(); + } + else + { + if(encoder.IsConnected) + throw new InvalidOperationException("The given encoder is already connected to another ArchiveWriter."); + + if(mEncoder != null) + mEncoder.Disconnect(); + + mEncoder = encoder; + mEncoder.Connect(this); + } + } + + #endregion + + #region Private Helper Methods + + private void WriteHeaderInfo(BinaryWriter writer, long offset, long size, uint crc) + { + uint infoCRC = CRC.kInitCRC; + infoCRC = CRC.Update(infoCRC, offset); + infoCRC = CRC.Update(infoCRC, size); + infoCRC = CRC.Update(infoCRC, crc); + infoCRC = CRC.Finish(infoCRC); + + writer.Write(infoCRC); + writer.Write(offset); + writer.Write(size); + writer.Write(crc); + } + + private void WriteNumber(BinaryWriter writer, BlockType value) + { + WriteNumber(writer, (byte)value); + } + + private void WriteNumber(BinaryWriter writer, long number) + { + WriteNumber(writer, checked((ulong)number)); + } + + private void WriteNumber(BinaryWriter writer, ulong number) + { + // TODO: Use the short forms if applicable. + writer.Write((byte)0xFF); + writer.Write(number); + } + + private long CalculateHeaderLimit() + { + return 1024; // HACK: mFileSets.SelectMany(stream => stream.Files).ToArray() is too slow + + //const int kMaxNumberLen = 9; // 0xFF + sizeof(ulong) + //const int kBlockTypeLen = kMaxNumberLen; + //const int kZeroNumberLen = kMaxNumberLen; + //long limit = 0; + + //var files = mFileSets.SelectMany(stream => stream.Files).ToArray(); + //if(files.Length != 0) + //{ + // limit += kBlockTypeLen; // BlockType.Header + + // var inputStreams = mFileSets.SelectMany(fileset => fileset.InputStreams).ToArray(); + // if(inputStreams.Any(stream => stream.Size != 0)) + // { + // limit += kBlockTypeLen; // BlockType.MainStreamsInfo + // limit += kBlockTypeLen; // BlockType.PackInfo + // limit += kZeroNumberLen; // zero = offset to input streams + // limit += kMaxNumberLen; // inputStreams.Length + // limit += kBlockTypeLen; //BlockType.Size + // limit += inputStreams.Length * kMaxNumberLen; // inputStreams: inputStream.Size + // limit += kBlockTypeLen; // BlockType.End + // limit += kBlockTypeLen; // BlockType.UnpackInfo + // limit += kBlockTypeLen; // BlockType.Folder + // limit += kMaxNumberLen; // mFileSets.Count + // limit += kZeroNumberLen; // zero = inline data + // foreach(var fileset in mFileSets) + // { + // limit += kMaxNumberLen; // fileset.Coders.Length + // foreach(var coder in fileset.Coders) + // { + // limit += 1; // flags + // limit += coder.MethodId.GetLength(); // coder.MethodId + + // if(coder.InputStreams.Length != 1 || coder.OutputStreams.Length != 1) + // { + // limit += kMaxNumberLen; // coder.InputStreams.Length + // limit += kMaxNumberLen; // coder.OutputStreams.Length + // } + + // if(coder.Settings != null) + // { + // limit += kMaxNumberLen; // coder.Settings.Length + // limit += coder.Settings.Length; // coder.Settings + // } + + // // TODO: Bind pairs and association to streams ... + // if(fileset.Coders.Length > 1 || coder.InputStreams.Length != 1 || coder.OutputStreams.Length != 1) + // throw new NotSupportedException(); + // } + // } + // limit += kBlockTypeLen; // BlockType.CodersUnpackSize + // limit += mFileSets.Count * kMaxNumberLen; // mFileSets: fileset.DataStream.GetSize(fileset) + // limit += kBlockTypeLen; // BlockType.End + // limit += kBlockTypeLen; // BlockType.SubStreamsInfo + // limit += kBlockTypeLen; // BlockType.NumUnpackStream + // limit += mFileSets.Count * kMaxNumberLen; // mFileSets: stream.Files.Length + // limit += kBlockTypeLen; // BlockType.Size + // limit += mFileSets.Sum(fileset => fileset.Files.Length - 1) * kMaxNumberLen; // mFileSets: fileset.Files[0..n-1]: stream.Files[i].Size + // limit += kBlockTypeLen; // BlockType.End + // limit += kBlockTypeLen; // BlockType.End + // } + + // limit += kBlockTypeLen; // BlockType.FilesInfo + // limit += kMaxNumberLen; // files.Length + + // limit += kBlockTypeLen; // BlockType.Name + // limit += kMaxNumberLen; // 1 + files.Sum(file => file.Name.Length + 1) * 2 + // limit += kZeroNumberLen; // zero = inline names + // for(int i = 0; i < files.Length; i++) + // limit += (files[i].Name.Length + 1) * 2; + + // if(files.Any(file => file.Size == 0)) + // { + // limit += kBlockTypeLen; // BlockType.EmptyStream + // limit += kMaxNumberLen; // (files.Length + 7) / 8 + // limit += (files.Length + 7) / 8; // bit vector + // limit += kBlockTypeLen; // BlockType.EmptyFile + // limit += kMaxNumberLen; // (files.Length + 7) / 8 -- this is an upper bound, for an exact size we need to count the number of empty streams + // limit += (files.Length + 7) / 8; // bit vector + // } + + // limit += kBlockTypeLen; // BlockType.CTime + // limit += kMaxNumberLen; // (ctimeCount + 7) / 8 + 2 + ctimeCount * 8; + // limit += (files.Length + 7) / 8 + 2 + files.Length * 8; + + // limit += kBlockTypeLen; // BlockType.ATime + // limit += kMaxNumberLen; // (atimeCount + 7) / 8 + 2 + atimeCount * 8; + // limit += (files.Length + 7) / 8 + 2 + files.Length * 8; + + // limit += kBlockTypeLen; // BlockType.MTime + // limit += kMaxNumberLen; // (mtimeCount + 7) / 8 + 2 + mtimeCount * 8; + // limit += (files.Length + 7) / 8 + 2 + files.Length * 8; + + // limit += kBlockTypeLen; // BlockType.End + //} + + //return limit; + } + + #endregion + } + + public static class ArchiveWriterExtensions + { + #region ArchiveWriter Extensions + + public static void WriteFile(this ArchiveWriter writer, IArchiveWriterEntry metadata, Stream content) + { + if(writer.CurrentEncoder == null) + throw new InvalidOperationException("No current encoder configured."); + + using(var stream = writer.CurrentEncoder.BeginWriteFile(metadata)) + content.CopyTo(stream); + } + + public static void WriteFile(this ArchiveWriter writer, DirectoryInfo root, FileInfo file) + { + using(var content = file.OpenRead()) + writer.WriteFile(new FileBasedArchiveWriterEntry(root, file), content); + } + + public static void WriteFiles(this ArchiveWriter writer, DirectoryInfo root, IEnumerable files) + { + foreach(var file in files) + writer.WriteFile(root, file); + } + + public static void WriteFiles(this ArchiveWriter writer, DirectoryInfo root, params FileInfo[] files) + { + writer.WriteFiles(root, (IEnumerable)files); + } + + #endregion + } +} diff --git a/SharpCompress/Common/SevenZip/CBindPair.cs b/SharpCompress/Common/SevenZip/CBindPair.cs new file mode 100644 index 00000000..20831537 --- /dev/null +++ b/SharpCompress/Common/SevenZip/CBindPair.cs @@ -0,0 +1,8 @@ +namespace SharpCompress.Common.SevenZip +{ + internal class CBindPair + { + internal int InIndex; + internal int OutIndex; + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/CCoderInfo.cs b/SharpCompress/Common/SevenZip/CCoderInfo.cs new file mode 100644 index 00000000..d3a38ff2 --- /dev/null +++ b/SharpCompress/Common/SevenZip/CCoderInfo.cs @@ -0,0 +1,10 @@ +namespace SharpCompress.Common.SevenZip +{ + internal class CCoderInfo + { + internal CMethodId MethodId; + internal byte[] Props; + internal int NumInStreams; + internal int NumOutStreams; + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/CFileItem.cs b/SharpCompress/Common/SevenZip/CFileItem.cs new file mode 100644 index 00000000..3940302a --- /dev/null +++ b/SharpCompress/Common/SevenZip/CFileItem.cs @@ -0,0 +1,34 @@ +using System; + +namespace SharpCompress.Common.SevenZip +{ + public class CFileItem + { + public long Size { get; internal set; } + public uint? Attrib { get; internal set; } + public uint? Crc { get; internal set; } + public string Name { get; internal set; } + + public bool HasStream { get; internal set; } + public bool IsDir { get; internal set; } + public bool CrcDefined { get { return Crc != null; } } + public bool AttribDefined { get { return Attrib != null; } } + + public void SetAttrib(uint attrib) + { + this.Attrib = attrib; + } + + public DateTime? CTime { get; internal set; } + public DateTime? ATime { get; internal set; } + public DateTime? MTime { get; internal set; } + + public long? StartPos { get; internal set; } + public bool IsAnti { get; internal set; } + + internal CFileItem() + { + HasStream = true; + } + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/CFolder.cs b/SharpCompress/Common/SevenZip/CFolder.cs new file mode 100644 index 00000000..3394a755 --- /dev/null +++ b/SharpCompress/Common/SevenZip/CFolder.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using ManagedLzma.LZMA.Master.SevenZip; +using SharpCompress.Compressor.LZMA; + +namespace SharpCompress.Common.SevenZip +{ + internal class CFolder + { + internal List Coders = new List(); + internal List BindPairs = new List(); + internal List PackStreams = new List(); + internal int FirstPackStreamId; + internal List UnpackSizes = new List(); + internal uint? UnpackCRC; + internal bool UnpackCRCDefined { get { return UnpackCRC != null; } } + + public long GetUnpackSize() + { + if(UnpackSizes.Count == 0) + return 0; + + for(int i = UnpackSizes.Count - 1; i >= 0; i--) + if(FindBindPairForOutStream(i) < 0) + return UnpackSizes[i]; + + throw new Exception(); + } + + public int GetNumOutStreams() + { + int count = 0; + for(int i = 0; i < Coders.Count; i++) + count += Coders[i].NumOutStreams; + + return count; + } + + public int FindBindPairForInStream(int inStreamIndex) + { + for(int i = 0; i < BindPairs.Count; i++) + if(BindPairs[i].InIndex == inStreamIndex) + return i; + + return -1; + } + + public int FindBindPairForOutStream(int outStreamIndex) + { + for(int i = 0; i < BindPairs.Count; i++) + if(BindPairs[i].OutIndex == outStreamIndex) + return i; + + return -1; + } + + public int FindPackStreamArrayIndex(int inStreamIndex) + { + for(int i = 0; i < PackStreams.Count; i++) + if(PackStreams[i] == inStreamIndex) + return i; + + return -1; + } + + public bool IsEncrypted() + { + for(int i = Coders.Count - 1; i >= 0; i--) + if(Coders[i].MethodId == CMethodId.kAES) + return true; + + return false; + } + + public bool CheckStructure() + { + const int kNumCodersMax = 32; // don't change it + const int kMaskSize = 32; // it must be >= kNumCodersMax + const int kNumBindsMax = 32; + + if(Coders.Count > kNumCodersMax || BindPairs.Count > kNumBindsMax) + return false; + + { + var v = new BitVector(BindPairs.Count + PackStreams.Count); + + for(int i = 0; i < BindPairs.Count; i++) + if(v.GetAndSet(BindPairs[i].InIndex)) + return false; + + for(int i = 0; i < PackStreams.Count; i++) + if(v.GetAndSet(PackStreams[i])) + return false; + } + + { + var v = new BitVector(UnpackSizes.Count); + for(int i = 0; i < BindPairs.Count; i++) + if(v.GetAndSet(BindPairs[i].OutIndex)) + return false; + } + + uint[] mask = new uint[kMaskSize]; + + { + List inStreamToCoder = new List(); + List outStreamToCoder = new List(); + for(int i = 0; i < Coders.Count; i++) + { + CCoderInfo coder = Coders[i]; + for(int j = 0; j < coder.NumInStreams; j++) + inStreamToCoder.Add(i); + for(int j = 0; j < coder.NumOutStreams; j++) + outStreamToCoder.Add(i); + } + + for(int i = 0; i < BindPairs.Count; i++) + { + CBindPair bp = BindPairs[i]; + mask[inStreamToCoder[bp.InIndex]] |= (1u << outStreamToCoder[bp.OutIndex]); + } + } + + for(int i = 0; i < kMaskSize; i++) + for(int j = 0; j < kMaskSize; j++) + if(((1u << j) & mask[i]) != 0) + mask[i] |= mask[j]; + + for(int i = 0; i < kMaskSize; i++) + if(((1u << i) & mask[i]) != 0) + return false; + + return true; + } + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/CMethodId.cs b/SharpCompress/Common/SevenZip/CMethodId.cs new file mode 100644 index 00000000..e0944d38 --- /dev/null +++ b/SharpCompress/Common/SevenZip/CMethodId.cs @@ -0,0 +1,55 @@ +namespace SharpCompress.Common.SevenZip +{ + internal struct CMethodId + { + public const ulong kCopyId = 0; + public const ulong kLzmaId = 0x030101; + public const ulong kLzma2Id = 0x21; + public const ulong kAESId = 0x06F10701; + + public static readonly CMethodId kCopy = new CMethodId(kCopyId); + public static readonly CMethodId kLzma = new CMethodId(kLzmaId); + public static readonly CMethodId kLzma2 = new CMethodId(kLzma2Id); + public static readonly CMethodId kAES = new CMethodId(kAESId); + + public readonly ulong Id; + + public CMethodId(ulong id) + { + this.Id = id; + } + + public override int GetHashCode() + { + return Id.GetHashCode(); + } + + public override bool Equals(object obj) + { + return obj is CMethodId && (CMethodId)obj == this; + } + + public bool Equals(CMethodId other) + { + return Id == other.Id; + } + + public static bool operator ==(CMethodId left, CMethodId right) + { + return left.Id == right.Id; + } + + public static bool operator !=(CMethodId left, CMethodId right) + { + return left.Id != right.Id; + } + + public int GetLength() + { + int bytes = 0; + for(ulong value = Id; value != 0; value >>= 8) + bytes++; + return bytes; + } + } +} diff --git a/SharpCompress/Common/SevenZip/CStreamSwitch.cs b/SharpCompress/Common/SevenZip/CStreamSwitch.cs new file mode 100644 index 00000000..af837932 --- /dev/null +++ b/SharpCompress/Common/SevenZip/CStreamSwitch.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SharpCompress.Compressor.LZMA; + +namespace SharpCompress.Common.SevenZip +{ + internal struct CStreamSwitch: IDisposable + { + private ArchiveReader _archive; + private bool _needRemove; + private bool _active; + + public void Dispose() + { + if(_active) + { + _active = false; + Log.WriteLine("[end of switch]"); + } + + if(_needRemove) + { + _needRemove = false; + _archive.DeleteByteStream(); + } + } + + public void Set(ArchiveReader archive, byte[] dataVector) + { + Dispose(); + _archive = archive; + _archive.AddByteStream(dataVector, 0, dataVector.Length); + _needRemove = true; + _active = true; + } + + public void Set(ArchiveReader archive, List dataVector) + { + Dispose(); + _active = true; + + byte external = archive.ReadByte(); + if(external != 0) + { + int dataIndex = archive.ReadNum(); + if(dataIndex < 0 || dataIndex >= dataVector.Count) + throw new InvalidOperationException(); + + Log.WriteLine("[switch to stream {0}]", dataIndex); + _archive = archive; + _archive.AddByteStream(dataVector[dataIndex], 0, dataVector[dataIndex].Length); + _needRemove = true; + _active = true; + } + else + { + Log.WriteLine("[inline data]"); + } + } + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/DataReader.cs b/SharpCompress/Common/SevenZip/DataReader.cs new file mode 100644 index 00000000..34e43bc3 --- /dev/null +++ b/SharpCompress/Common/SevenZip/DataReader.cs @@ -0,0 +1,168 @@ +using System; +using System.IO; +using System.Text; +using SharpCompress.Compressor.LZMA; + +namespace SharpCompress.Common.SevenZip +{ + internal class DataReader + { + #region Static Methods + + public static uint Get32(byte[] buffer, int offset) + { + return (uint)buffer[offset] + + ((uint)buffer[offset + 1] << 8) + + ((uint)buffer[offset + 2] << 16) + + ((uint)buffer[offset + 3] << 24); + } + + public static ulong Get64(byte[] buffer, int offset) + { + return (ulong)buffer[offset] + + ((ulong)buffer[offset + 1] << 8) + + ((ulong)buffer[offset + 2] << 16) + + ((ulong)buffer[offset + 3] << 24) + + ((ulong)buffer[offset + 4] << 32) + + ((ulong)buffer[offset + 5] << 40) + + ((ulong)buffer[offset + 6] << 48) + + ((ulong)buffer[offset + 7] << 56); + } + + #endregion + + #region Variables + + private byte[] _buffer; + private int _origin; + private int _offset; + private int _ending; + + #endregion + + #region Public Methods + + public DataReader(byte[] buffer, int offset, int length) + { + _buffer = buffer; + _origin = offset; + _offset = offset; + _ending = offset + length; + } + + public int Offset + { + get { return _offset; } + } + + public Byte ReadByte() + { + if(_offset >= _ending) + throw new EndOfStreamException(); + + return _buffer[_offset++]; + } + + public void ReadBytes(byte[] buffer, int offset, int length) + { + if(length > _ending - _offset) + throw new EndOfStreamException(); + + while(length-- > 0) + buffer[offset++] = _buffer[_offset++]; + } + + public void SkipData(long size) + { + if(size > _ending - _offset) + throw new EndOfStreamException(); + + _offset += (int)size; + Log.WriteLine("SkipData {0}", size); + } + + public void SkipData() + { + SkipData(checked((long)ReadNumber())); + } + + public ulong ReadNumber() + { + if(_offset >= _ending) + throw new EndOfStreamException(); + + byte firstByte = _buffer[_offset++]; + byte mask = 0x80; + ulong value = 0; + + for(int i = 0; i < 8; i++) + { + if((firstByte & mask) == 0) + { + ulong highPart = firstByte & (mask - 1u); + value += highPart << (i * 8); + return value; + } + + if(_offset >= _ending) + throw new EndOfStreamException(); + + value |= (ulong)_buffer[_offset++] << (8 * i); + mask >>= 1; + } + + return value; + } + + public int ReadNum() + { + ulong value = ReadNumber(); + if(value > Int32.MaxValue) + throw new NotSupportedException(); + + return (int)value; + } + + public uint ReadUInt32() + { + if(_offset + 4 > _ending) + throw new EndOfStreamException(); + + uint res = Get32(_buffer, _offset); + _offset += 4; + return res; + } + + public ulong ReadUInt64() + { + if(_offset + 8 > _ending) + throw new EndOfStreamException(); + + ulong res = Get64(_buffer, _offset); + _offset += 8; + return res; + } + + public string ReadString() + { + int ending = _offset; + + for(; ; ) + { + if(ending + 2 > _ending) + throw new EndOfStreamException(); + + if(_buffer[ending] == 0 && _buffer[ending + 1] == 0) + break; + + ending += 2; + } + + string str = Encoding.Unicode.GetString(_buffer, _offset, ending - _offset); + _offset = ending + 2; + return str; + } + + #endregion + } +} \ No newline at end of file diff --git a/SharpCompress/Common/SevenZip/SevenZipEntry.cs b/SharpCompress/Common/SevenZip/SevenZipEntry.cs index 9b51f250..00f448b6 100644 --- a/SharpCompress/Common/SevenZip/SevenZipEntry.cs +++ b/SharpCompress/Common/SevenZip/SevenZipEntry.cs @@ -22,7 +22,7 @@ namespace SharpCompress.Common.SevenZip public override uint Crc { - get { return (uint)FilePart.Header.FileCRC; } + get { return FilePart.Header.Crc ?? 0; } } public override string FilePath @@ -67,7 +67,7 @@ namespace SharpCompress.Common.SevenZip public override bool IsDirectory { - get { return FilePart.Header.IsDirectory; } + get { return FilePart.Header.IsDir; } } public override bool IsSplit diff --git a/SharpCompress/Common/SevenZip/SevenZipFilePart.cs b/SharpCompress/Common/SevenZip/SevenZipFilePart.cs index 5cd02bc1..5b1e5449 100644 --- a/SharpCompress/Common/SevenZip/SevenZipFilePart.cs +++ b/SharpCompress/Common/SevenZip/SevenZipFilePart.cs @@ -1,4 +1,5 @@ -using System.IO; +using System; +using System.IO; using System.Linq; using SharpCompress.IO; @@ -7,38 +8,56 @@ namespace SharpCompress.Common.SevenZip internal class SevenZipFilePart : FilePart { private CompressionType? type; + private Stream stream; + private ArchiveDatabase database; - internal SevenZipFilePart(SevenZipHeaderFactory factory, int index, HeaderEntry fileEntry, Stream stream) + internal SevenZipFilePart(Stream stream, ArchiveDatabase database, int index, CFileItem fileEntry) { + this.stream = stream; + this.database = database; + Index = index; Header = fileEntry; - this.BaseStream = stream; + if (Header.HasStream) + { + Folder = database.Folders[database.FileIndexToFolderIndexMap[index]]; + } } internal Stream BaseStream { get; private set; } - internal HeaderEntry Header { get; set; } + internal CFileItem Header { get; private set; } + internal CFolder Folder { get; private set; } + internal int Index { get; private set; } internal override string FilePartName { get { return Header.Name; } } + internal override Stream GetRawStream() + { + return null; + } + internal override Stream GetCompressedStream() { if (!Header.HasStream) { return null; } - var stream = Header.Folder.GetStream(); - if (Header.FolderOffset > 0) - { - stream.Skip((long)Header.FolderOffset); - } - return new ReadOnlySubStream(stream, (long)Header.Size); - } + var folderStream = database.GetFolderStream(stream, Folder, null); - internal override Stream GetRawStream() - { - return null; + int firstFileIndex = database.FolderStartFileIndex[database.Folders.IndexOf(Folder)]; + int skipCount = Index - firstFileIndex; + long skipSize = 0; + for (int i = 0; i < skipCount; i++) + { + skipSize += database.Files[firstFileIndex + i].Size; + } + if (skipSize > 0) + { + folderStream.Skip(skipSize); + } + return new ReadOnlySubStream(folderStream, Header.Size); } public CompressionType CompressionType @@ -47,10 +66,44 @@ namespace SharpCompress.Common.SevenZip { if (type == null) { - this.type = Header.Folder.GetCompressions().First().CompressionType; + type = GetCompression(); } return type.Value; } } + + //copied from DecoderRegistry + const uint k_Copy = 0x0; + const uint k_Delta = 3; + const uint k_LZMA2 = 0x21; + const uint k_LZMA = 0x030101; + const uint k_PPMD = 0x030401; + const uint k_BCJ = 0x03030103; + const uint k_BCJ2 = 0x0303011B; + const uint k_Deflate = 0x040108; + const uint k_BZip2 = 0x040202; + + internal CompressionType GetCompression() + { + var coder = Folder.Coders.First(); + switch (coder.MethodId.Id) + { + case k_LZMA: + case k_LZMA2: + { + return CompressionType.LZMA; + } + case k_PPMD: + { + return CompressionType.PPMd; + } + case k_BZip2: + { + return CompressionType.BZip2; + } + default: + throw new NotImplementedException(); + } + } } } diff --git a/SharpCompress/Compressor/LZMA/AesDecoderStream.cs b/SharpCompress/Compressor/LZMA/AesDecoderStream.cs new file mode 100644 index 00000000..3f58af86 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/AesDecoderStream.cs @@ -0,0 +1,228 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using SharpCompress.Compressor.LZMA.Utilites; + +namespace SharpCompress.Compressor.LZMA +{ + internal class AesDecoderStream: DecoderStream2 + { + #region Variables + + private Stream mStream; + private ICryptoTransform mDecoder; + private byte[] mBuffer; + private long mWritten; + private long mLimit; + private int mOffset; + private int mEnding; + private int mUnderflow; + + #endregion + + #region Stream Methods + + public AesDecoderStream(Stream input, byte[] info, IPasswordProvider pass, long limit) + { + mStream = input; + mLimit = limit; + + if(((uint)input.Length & 15) != 0) + throw new NotSupportedException("AES decoder does not support padding."); + + int numCyclesPower; + byte[] salt, seed; + Init(info, out numCyclesPower, out salt, out seed); + + byte[] password = Encoding.Unicode.GetBytes(pass.CryptoGetTextPassword()); + byte[] key = InitKey(numCyclesPower, salt, password); + + using(var aes = Aes.Create()) + { + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + mDecoder = aes.CreateDecryptor(key, seed); + } + + mBuffer = new byte[4 << 10]; + } + + protected override void Dispose(bool disposing) + { + try + { + if(disposing) + { + mStream.Dispose(); + mDecoder.Dispose(); + } + } + finally + { + base.Dispose(disposing); + } + } + + public override long Position + { + get { return mWritten; } + } + + public override long Length + { + get { return mLimit; } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if(count == 0 || mWritten == mLimit) + return 0; + + if(mUnderflow > 0) + return HandleUnderflow(buffer, offset, count); + + // Need at least 16 bytes to proceed. + if(mEnding - mOffset < 16) + { + Buffer.BlockCopy(mBuffer, mOffset, mBuffer, 0, mEnding - mOffset); + mEnding -= mOffset; + mOffset = 0; + + do + { + int read = mStream.Read(mBuffer, mEnding, mBuffer.Length - mEnding); + if(read == 0) + { + // We are not done decoding and have less than 16 bytes. + throw new EndOfStreamException(); + } + + mEnding += read; + } + while(mEnding - mOffset < 16); + } + + // We shouldn't return more data than we are limited to. + // Currently this is handled by forcing an underflow if + // the stream length is not a multiple of the block size. + if(count > mLimit - mWritten) + count = (int)(mLimit - mWritten); + + // We cannot transform less than 16 bytes into the target buffer, + // but we also cannot return zero, so we need to handle this. + // We transform the data locally and use our own buffer as cache. + if(count < 16) + return HandleUnderflow(buffer, offset, count); + + if(count > mEnding - mOffset) + count = mEnding - mOffset; + + // Otherwise we transform directly into the target buffer. + int processed = mDecoder.TransformBlock(mBuffer, mOffset, count & ~15, buffer, offset); + mOffset += processed; + mWritten += processed; + return processed; + } + + #endregion + + #region Private Methods + + private void Init(byte[] info, out int numCyclesPower, out byte[] salt, out byte[] iv) + { + byte bt = info[0]; + numCyclesPower = bt & 0x3F; + + if((bt & 0xC0) == 0) + { + salt = new byte[0]; + iv = new byte[0]; + return; + } + + int saltSize = (bt >> 7) & 1; + int ivSize = (bt >> 6) & 1; + if(info.Length == 1) + throw new InvalidOperationException(); + + byte bt2 = info[1]; + saltSize += (bt2 >> 4); + ivSize += (bt2 & 15); + if(info.Length < 2 + saltSize + ivSize) + throw new InvalidOperationException(); + + salt = new byte[saltSize]; + for(int i = 0; i < saltSize; i++) + salt[i] = info[i + 2]; + + iv = new byte[16]; + for(int i = 0; i < ivSize; i++) + iv[i] = info[i + saltSize + 2]; + + if(numCyclesPower > 24) + throw new NotSupportedException(); + } + + private byte[] InitKey(int mNumCyclesPower, byte[] salt, byte[] pass) + { + if(mNumCyclesPower == 0x3F) + { + var key = new byte[32]; + + int pos; + for(pos = 0; pos < salt.Length; pos++) + key[pos] = salt[pos]; + for(int i = 0; i < pass.Length && pos < 32; i++) + key[pos++] = pass[i]; + + return key; + } + else + { + using(var sha = System.Security.Cryptography.SHA256.Create()) + { + byte[] counter = new byte[8]; + long numRounds = 1L << mNumCyclesPower; + for(long round = 0; round < numRounds; round++) + { + sha.TransformBlock(salt, 0, salt.Length, null, 0); + sha.TransformBlock(pass, 0, pass.Length, null, 0); + sha.TransformBlock(counter, 0, 8, null, 0); + + // This mirrors the counter so we don't have to convert long to byte[] each round. + // (It also ensures the counter is little endian, which BitConverter does not.) + for(int i = 0; i < 8; i++) + if(++counter[i] != 0) + break; + } + + sha.TransformFinalBlock(counter, 0, 0); + return sha.Hash; + } + } + } + + private int HandleUnderflow(byte[] buffer, int offset, int count) + { + // If this is zero we were called to create a new underflow buffer. + // Just transform as much as possible so we can feed from it as long as possible. + if(mUnderflow == 0) + { + int blockSize = (mEnding - mOffset) & ~15; + mUnderflow = mDecoder.TransformBlock(mBuffer, mOffset, blockSize, mBuffer, mOffset); + } + + if(count > mUnderflow) + count = mUnderflow; + + Buffer.BlockCopy(mBuffer, mOffset, buffer, offset, count); + mWritten += count; + mOffset += count; + mUnderflow -= count; + return count; + } + + #endregion + } +} diff --git a/SharpCompress/Compressor/LZMA/Bcj2DecoderStream.cs b/SharpCompress/Compressor/LZMA/Bcj2DecoderStream.cs new file mode 100644 index 00000000..100c0912 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Bcj2DecoderStream.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace SharpCompress.Compressor.LZMA +{ + class Bcj2DecoderStream: DecoderStream2 + { + const int kNumTopBits = 24; + const uint kTopValue = (1 << kNumTopBits); + + private class RangeDecoder + { + internal Stream mStream; + internal uint Range; + internal uint Code; + + public RangeDecoder(Stream stream) + { + mStream = stream; + Range = 0xFFFFFFFF; + for(int i = 0; i < 5; i++) + Code = (Code << 8) | ReadByte(); + } + + public byte ReadByte() + { + int bt = mStream.ReadByte(); + if(bt < 0) + throw new EndOfStreamException(); + + return (byte)bt; + } + + public void Dispose() + { + mStream.Dispose(); + } + } + + private class StatusDecoder + { + private const int numMoveBits = 5; + + private const int kNumBitModelTotalBits = 11; + private const uint kBitModelTotal = 1u << kNumBitModelTotalBits; + + private uint Prob; + + public StatusDecoder() + { + Prob = kBitModelTotal / 2; + } + + private void UpdateModel(uint symbol) + { + /* + Prob -= (Prob + ((symbol - 1) & ((1 << numMoveBits) - 1))) >> numMoveBits; + Prob += (1 - symbol) << (kNumBitModelTotalBits - numMoveBits); + */ + if(symbol == 0) + Prob += (kBitModelTotal - Prob) >> numMoveBits; + else + Prob -= (Prob) >> numMoveBits; + } + + public uint Decode(RangeDecoder decoder) + { + uint newBound = (decoder.Range >> kNumBitModelTotalBits) * Prob; + if(decoder.Code < newBound) + { + decoder.Range = newBound; + Prob += (kBitModelTotal - Prob) >> numMoveBits; + if(decoder.Range < kTopValue) + { + decoder.Code = (decoder.Code << 8) | decoder.ReadByte(); + decoder.Range <<= 8; + } + return 0; + } + else + { + decoder.Range -= newBound; + decoder.Code -= newBound; + Prob -= Prob >> numMoveBits; + if(decoder.Range < kTopValue) + { + decoder.Code = (decoder.Code << 8) | decoder.ReadByte(); + decoder.Range <<= 8; + } + return 1; + } + } + } + + private Stream mMainStream; + private Stream mCallStream; + private Stream mJumpStream; + private RangeDecoder mRangeDecoder; + private StatusDecoder[] mStatusDecoder; + private long mWritten; + private long mLimit; + private IEnumerator mIter; + private bool mFinished; + + public Bcj2DecoderStream(Stream[] streams, byte[] info, long limit) + { + if(info != null && info.Length > 0) + throw new NotSupportedException(); + + if(streams.Length != 4) + throw new NotSupportedException(); + + mLimit = limit; + mMainStream = streams[0]; + mCallStream = streams[1]; + mJumpStream = streams[2]; + mRangeDecoder = new RangeDecoder(streams[3]); + + mStatusDecoder = new StatusDecoder[256 + 2]; + for(int i = 0; i < mStatusDecoder.Length; i++) + mStatusDecoder[i] = new StatusDecoder(); + + mIter = Run().GetEnumerator(); + } + + private static bool IsJcc(byte b0, byte b1) + { + return b0 == 0x0F + && (b1 & 0xF0) == 0x80; + } + + private static bool IsJ(byte b0, byte b1) + { + return (b1 & 0xFE) == 0xE8 + || IsJcc(b0, b1); + } + + private static int GetIndex(byte b0, byte b1) + { + if(b1 == 0xE8) + return b0; + else if(b1 == 0xE9) + return 256; + else + return 257; + } + + public override int Read(byte[] buffer, int offset, int count) + { + if(count == 0 || mFinished) + return 0; + + for(int i = 0; i < count; i++) + { + if(!mIter.MoveNext()) + { + mFinished = true; + return i; + } + + buffer[offset + i] = mIter.Current; + } + + return count; + } + + public IEnumerable Run() + { + const uint kBurstSize = (1u << 18); + + byte prevByte = 0; + uint processedBytes = 0; + for(; ; ) + { + byte b = 0; + uint i; + for(i = 0; i < kBurstSize; i++) + { + int tmp = mMainStream.ReadByte(); + if(tmp < 0) + yield break; + + b = (byte)tmp; + mWritten++; yield return b; + if(IsJ(prevByte, b)) + break; + + prevByte = b; + } + + processedBytes += i; + if(i == kBurstSize) + continue; + + if(mStatusDecoder[GetIndex(prevByte, b)].Decode(mRangeDecoder) == 1) + { + Stream s = (b == 0xE8) ? mCallStream : mJumpStream; + + uint src = 0; + for(i = 0; i < 4; i++) + { + int b0 = s.ReadByte(); + if(b0 < 0) + throw new EndOfStreamException(); + + src <<= 8; + src |= (uint)b0; + } + + uint dest = src - (uint)(mWritten + 4); + mWritten++; yield return (byte)dest; + mWritten++; yield return (byte)(dest >> 8); + mWritten++; yield return (byte)(dest >> 16); + mWritten++; yield return (byte)(dest >> 24); + prevByte = (byte)(dest >> 24); + processedBytes += 4; + } + else + { + prevByte = b; + } + } + } + } +} diff --git a/SharpCompress/Compressor/LZMA/BitVector.cs b/SharpCompress/Compressor/LZMA/BitVector.cs new file mode 100644 index 00000000..350efbea --- /dev/null +++ b/SharpCompress/Compressor/LZMA/BitVector.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SharpCompress.Compressor.LZMA +{ + public class BitVector + { + private uint[] mBits; + private int mLength; + + public BitVector(int length) + { + mLength = length; + mBits = new uint[(length + 31) >> 5]; + } + + public BitVector(int length, bool initValue) + { + mLength = length; + mBits = new uint[(length + 31) >> 5]; + + if(initValue) + for(int i = 0; i < mBits.Length; i++) + mBits[i] = ~0u; + } + + public BitVector(List bits) + : this(bits.Count) + { + for(int i = 0; i < bits.Count; i++) + if(bits[i]) + SetBit(i); + } + + public bool[] ToArray() + { + bool[] bits = new bool[mLength]; + for(int i = 0; i < bits.Length; i++) + bits[i] = this[i]; + return bits; + } + + public int Length + { + get { return mLength; } + } + + public bool this[int index] + { + get + { + if(index < 0 || index >= mLength) + throw new ArgumentOutOfRangeException("index"); + + return (mBits[index >> 5] & (1u << (index & 31))) != 0; + } + } + + public void SetBit(int index) + { + if(index < 0 || index >= mLength) + throw new ArgumentOutOfRangeException("index"); + + mBits[index >> 5] |= 1u << (index & 31); + } + + internal bool GetAndSet(int index) + { + if(index < 0 || index >= mLength) + throw new ArgumentOutOfRangeException("index"); + + uint bits = mBits[index >> 5]; + uint mask = 1u << (index & 31); + mBits[index >> 5] |= mask; + return (bits & mask) != 0; + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(mLength); + for(int i = 0; i < mLength; i++) + sb.Append(this[i] ? 'x' : '.'); + return sb.ToString(); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/CRC.cs b/SharpCompress/Compressor/LZMA/CRC.cs index d7bac3c2..4b4f9f11 100644 --- a/SharpCompress/Compressor/LZMA/CRC.cs +++ b/SharpCompress/Compressor/LZMA/CRC.cs @@ -1,53 +1,111 @@ -namespace SharpCompress.Compressor.LZMA +using System; +using System.IO; + +namespace ManagedLzma.LZMA.Master.SevenZip { - internal class CRC + internal static class CRC { - public static readonly uint[] Table; + public const uint kInitCRC = 0xFFFFFFFF; + private static uint[] kTable = new uint[4 * 256]; static CRC() { - Table = new uint[256]; - const uint kPoly = 0xEDB88320; - for (uint i = 0; i < 256; i++) + const uint kCrcPoly = 0xEDB88320; + + for(uint i = 0; i < 256; i++) { uint r = i; - for (int j = 0; j < 8; j++) - if ((r & 1) != 0) - r = (r >> 1) ^ kPoly; - else - r >>= 1; - Table[i] = r; + for(int j = 0; j < 8; j++) + r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + + kTable[i] = r; + } + + for(uint i = 256; i < kTable.Length; i++) + { + uint r = kTable[i - 256]; + kTable[i] = kTable[r & 0xFF] ^ (r >> 8); } } - uint _value = 0xFFFFFFFF; - - public void Init() { _value = 0xFFFFFFFF; } - - public void UpdateByte(byte b) + public static uint From(Stream stream, long length) { - _value = Table[(((byte)(_value)) ^ b)] ^ (_value >> 8); + uint crc = kInitCRC; + byte[] buffer = new byte[Math.Min(length, 4 << 10)]; + while(length > 0) + { + int delta = stream.Read(buffer, 0, (int)Math.Min(length, buffer.Length)); + if(delta == 0) + throw new EndOfStreamException(); + crc = Update(crc, buffer, 0, delta); + length -= delta; + } + return Finish(crc); } - public void Update(byte[] data, uint offset, uint size) + public static uint Finish(uint crc) { - for (uint i = 0; i < size; i++) - _value = Table[(((byte)(_value)) ^ data[offset + i])] ^ (_value >> 8); + return ~crc; } - public uint GetDigest() { return _value ^ 0xFFFFFFFF; } - - static uint CalculateDigest(byte[] data, uint offset, uint size) + public static uint Update(uint crc, byte bt) { - CRC crc = new CRC(); - // crc.Init(); - crc.Update(data, offset, size); - return crc.GetDigest(); + return kTable[(crc & 0xFF) ^ bt] ^ (crc >> 8); } - static bool VerifyDigest(uint digest, byte[] data, uint offset, uint size) + public static uint Update(uint crc, uint value) { - return (CalculateDigest(data, offset, size) == digest); + crc ^= value; + return kTable[0x300 + (crc & 0xFF)] + ^ kTable[0x200 + ((crc >> 8) & 0xFF)] + ^ kTable[0x100 + ((crc >> 16) & 0xFF)] + ^ kTable[0x000 + (crc >> 24)]; } + + public static uint Update(uint crc, ulong value) + { + return Update(Update(crc, (uint)value), (uint)(value >> 32)); + } + + public static uint Update(uint crc, long value) + { + return Update(crc, (ulong)value); + } + + public static uint Update(uint crc, byte[] buffer, int offset, int length) + { + for(int i = 0; i < length; i++) + crc = Update(crc, buffer[offset + i]); + + return crc; + } + +#if !SILVERLIGHT && !PORTABLE + public static unsafe uint Update(uint crc, byte* buffer, int length) + { + while(length > 0 && ((int)buffer & 3) != 0) + { + crc = Update(crc, *buffer); + buffer++; + length--; + } + + while(length >= 4) + { + crc = Update(crc, *(uint*)buffer); + buffer += 4; + length -= 4; + } + + while(length > 0) + { + crc = Update(crc, *buffer); + length--; + } + + return crc; + } + +#endif } } diff --git a/SharpCompress/Compressor/LZMA/CRC2.cs b/SharpCompress/Compressor/LZMA/CRC2.cs new file mode 100644 index 00000000..d7bac3c2 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/CRC2.cs @@ -0,0 +1,53 @@ +namespace SharpCompress.Compressor.LZMA +{ + internal class CRC + { + public static readonly uint[] Table; + + static CRC() + { + Table = new uint[256]; + const uint kPoly = 0xEDB88320; + for (uint i = 0; i < 256; i++) + { + uint r = i; + for (int j = 0; j < 8; j++) + if ((r & 1) != 0) + r = (r >> 1) ^ kPoly; + else + r >>= 1; + Table[i] = r; + } + } + + uint _value = 0xFFFFFFFF; + + public void Init() { _value = 0xFFFFFFFF; } + + public void UpdateByte(byte b) + { + _value = Table[(((byte)(_value)) ^ b)] ^ (_value >> 8); + } + + public void Update(byte[] data, uint offset, uint size) + { + for (uint i = 0; i < size; i++) + _value = Table[(((byte)(_value)) ^ data[offset + i])] ^ (_value >> 8); + } + + public uint GetDigest() { return _value ^ 0xFFFFFFFF; } + + static uint CalculateDigest(byte[] data, uint offset, uint size) + { + CRC crc = new CRC(); + // crc.Init(); + crc.Update(data, offset, size); + return crc.GetDigest(); + } + + static bool VerifyDigest(uint digest, byte[] data, uint offset, uint size) + { + return (CalculateDigest(data, offset, size) == digest); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/DecoderStream.cs b/SharpCompress/Compressor/LZMA/DecoderStream.cs new file mode 100644 index 00000000..3009e2bc --- /dev/null +++ b/SharpCompress/Compressor/LZMA/DecoderStream.cs @@ -0,0 +1,173 @@ +using System; +using System.IO; +using SharpCompress.Common.SevenZip; +using SharpCompress.Compressor.LZMA.Utilites; + +namespace SharpCompress.Compressor.LZMA +{ + abstract class DecoderStream2: Stream + { + public override bool CanRead + { + get { return true; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return false; } + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override long Length + { + get { throw new NotImplementedException(); } + } + + public override long Position + { + get { throw new NotImplementedException(); } + set { throw new NotImplementedException(); } + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + } + + static class DecoderStreamHelper + { + private static int FindCoderIndexForOutStreamIndex(CFolder folderInfo, int outStreamIndex) + { + for(int coderIndex = 0; coderIndex < folderInfo.Coders.Count; coderIndex++) + { + var coderInfo = folderInfo.Coders[coderIndex]; + outStreamIndex -= coderInfo.NumOutStreams; + if(outStreamIndex < 0) + return coderIndex; + } + + throw new InvalidOperationException("Could not link output stream to coder."); + } + + private static void FindPrimaryOutStreamIndex(CFolder folderInfo, out int primaryCoderIndex, out int primaryOutStreamIndex) + { + bool foundPrimaryOutStream = false; + primaryCoderIndex = -1; + primaryOutStreamIndex = -1; + + for(int outStreamIndex = 0, coderIndex = 0; + coderIndex < folderInfo.Coders.Count; + coderIndex++) + { + for(int coderOutStreamIndex = 0; + coderOutStreamIndex < folderInfo.Coders[coderIndex].NumOutStreams; + coderOutStreamIndex++, outStreamIndex++) + { + if(folderInfo.FindBindPairForOutStream(outStreamIndex) < 0) + { + if(foundPrimaryOutStream) + throw new NotSupportedException("Multiple output streams."); + + foundPrimaryOutStream = true; + primaryCoderIndex = coderIndex; + primaryOutStreamIndex = outStreamIndex; + } + } + } + + if(!foundPrimaryOutStream) + throw new NotSupportedException("No output stream."); + } + + private static Stream CreateDecoderStream(Stream[] packStreams, long[] packSizes, Stream[] outStreams, CFolder folderInfo, int coderIndex, IPasswordProvider pass) + { + var coderInfo = folderInfo.Coders[coderIndex]; + if(coderInfo.NumOutStreams != 1) + throw new NotSupportedException("Multiple output streams are not supported."); + + int inStreamId = 0; + for(int i = 0; i < coderIndex; i++) + inStreamId += folderInfo.Coders[i].NumInStreams; + + int outStreamId = 0; + for(int i = 0; i < coderIndex; i++) + outStreamId += folderInfo.Coders[i].NumOutStreams; + + Stream[] inStreams = new Stream[coderInfo.NumInStreams]; + + for(int i = 0; i < inStreams.Length; i++, inStreamId++) + { + int bindPairIndex = folderInfo.FindBindPairForInStream(inStreamId); + if(bindPairIndex >= 0) + { + int pairedOutIndex = folderInfo.BindPairs[bindPairIndex].OutIndex; + + if(outStreams[pairedOutIndex] != null) + throw new NotSupportedException("Overlapping stream bindings are not supported."); + + int otherCoderIndex = FindCoderIndexForOutStreamIndex(folderInfo, pairedOutIndex); + inStreams[i] = CreateDecoderStream(packStreams, packSizes, outStreams, folderInfo, otherCoderIndex, pass); + //inStreamSizes[i] = folderInfo.UnpackSizes[pairedOutIndex]; + + if(outStreams[pairedOutIndex] != null) + throw new NotSupportedException("Overlapping stream bindings are not supported."); + + outStreams[pairedOutIndex] = inStreams[i]; + } + else + { + int index = folderInfo.FindPackStreamArrayIndex(inStreamId); + if(index < 0) + throw new NotSupportedException("Could not find input stream binding."); + + inStreams[i] = packStreams[index]; + //inStreamSizes[i] = packSizes[index]; + } + } + + long unpackSize = folderInfo.UnpackSizes[outStreamId]; + return DecoderRegistry.CreateDecoderStream(coderInfo.MethodId, inStreams, coderInfo.Props, pass, unpackSize); + } + + internal static Stream CreateDecoderStream(Stream inStream, long startPos, long[] packSizes, CFolder folderInfo, IPasswordProvider pass) + { + if(!folderInfo.CheckStructure()) + throw new NotSupportedException("Unsupported stream binding structure."); + + // We have multiple views into the same stream which will be used by several threads - need to sync those. + object sync = new object(); + Stream[] inStreams = new Stream[folderInfo.PackStreams.Count]; + for(int j = 0; j < folderInfo.PackStreams.Count; j++) + { + inStreams[j] = new SyncStreamView(sync, inStream, startPos, packSizes[j]); + startPos += packSizes[j]; + } + + Stream[] outStreams = new Stream[folderInfo.UnpackSizes.Count]; + + int primaryCoderIndex, primaryOutStreamIndex; + FindPrimaryOutStreamIndex(folderInfo, out primaryCoderIndex, out primaryOutStreamIndex); + return CreateDecoderStream(inStreams, packSizes, outStreams, folderInfo, primaryCoderIndex, pass); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Log.cs b/SharpCompress/Compressor/LZMA/Log.cs new file mode 100644 index 00000000..b67fda7c --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Log.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; + +namespace SharpCompress.Compressor.LZMA +{ + internal static class Log + { + private static Stack _indent = new Stack(); + private static bool _needsIndent = true; + + static Log() + { + _indent.Push(""); + } + + public static void PushIndent(string indent = " ") + { + _indent.Push(_indent.Peek() + indent); + } + + public static void PopIndent() + { + if(_indent.Count == 1) + throw new InvalidOperationException(); + + _indent.Pop(); + } + + private static void EnsureIndent() + { + if(_needsIndent) + { + _needsIndent = false; +#if !SILVERLIGHT && !PORTABLE + System.Diagnostics.Debug.Write(_indent.Peek()); +#endif + } + } + + public static void Write(object value) + { + EnsureIndent(); +#if !SILVERLIGHT && !PORTABLE + System.Diagnostics.Debug.Write(value); +#endif + } + + public static void Write(string text) + { + EnsureIndent(); +#if !SILVERLIGHT && !PORTABLE + System.Diagnostics.Debug.Write(text); +#endif + } + + public static void Write(string format, params object[] args) + { + EnsureIndent(); +#if !SILVERLIGHT && !PORTABLE + System.Diagnostics.Debug.Write(string.Format(format, args)); +#endif + } + + public static void WriteLine() + { + System.Diagnostics.Debug.WriteLine(""); + _needsIndent = true; + } + + public static void WriteLine(object value) + { + EnsureIndent(); + System.Diagnostics.Debug.WriteLine(value); + _needsIndent = true; + } + + public static void WriteLine(string text) + { + EnsureIndent(); + System.Diagnostics.Debug.WriteLine(text); + _needsIndent = true; + } + + public static void WriteLine(string format, params object[] args) + { + EnsureIndent(); + System.Diagnostics.Debug.WriteLine(string.Format(format, args)); + _needsIndent = true; + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Registry.cs b/SharpCompress/Compressor/LZMA/Registry.cs new file mode 100644 index 00000000..b2e038da --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Registry.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Linq; +using SharpCompress.Common.SevenZip; +using SharpCompress.Compressor.BZip2; +using SharpCompress.Compressor.Filters; +using SharpCompress.Compressor.LZMA.Utilites; +using SharpCompress.Compressor.PPMd; + +namespace SharpCompress.Compressor.LZMA +{ + internal static class DecoderRegistry + { + const uint k_Copy = 0x0; + const uint k_Delta = 3; + const uint k_LZMA2 = 0x21; + const uint k_LZMA = 0x030101; + const uint k_PPMD = 0x030401; + const uint k_BCJ = 0x03030103; + const uint k_BCJ2 = 0x0303011B; + const uint k_Deflate = 0x040108; + const uint k_BZip2 = 0x040202; + + internal static Stream CreateDecoderStream(CMethodId id, Stream[] inStreams, byte[] info, IPasswordProvider pass, long limit) + { + switch(id.Id) + { + case k_Copy: + if(info != null) + throw new NotSupportedException(); + return inStreams.Single(); + case k_LZMA: + case k_LZMA2: + return new LzmaStream(info, inStreams.Single(), -1, limit); +#if !SILVERLIGHT && !PORTABLE + case CMethodId.kAESId: + return new AesDecoderStream(inStreams.Single(), info, pass, limit); +#endif + case k_BCJ: + return new BCJFilter(false, inStreams.Single()); + case k_BCJ2: + return new Bcj2DecoderStream(inStreams, info, limit); + case k_BZip2: + return new BZip2Stream(inStreams.Single(), CompressionMode.Decompress, true); + case k_PPMD: + return new PpmdStream(new PpmdProperties(info), inStreams.Single(), false ); + default: + throw new NotSupportedException(); + } + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/CrcBuilderStream.cs b/SharpCompress/Compressor/LZMA/Utilites/CrcBuilderStream.cs new file mode 100644 index 00000000..97f0f882 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/CrcBuilderStream.cs @@ -0,0 +1,197 @@ +using System; +using System.IO; + +namespace SharpCompress.Compressor.LZMA.Utilites +{ + class CrcBuilderStream: Stream + { + private long mProcessed; + private Stream mTarget; + private uint mCRC; + private bool mFinished; + + public CrcBuilderStream(Stream target) + { + mTarget = target; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.kInitCRC; + } + + public long Processed + { + get { return mProcessed; } + } + + public uint Finish() + { + if(!mFinished) + { + mFinished = true; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Finish(mCRC); + } + + return mCRC; + } + + public override bool CanRead + { + get { return false; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return true; } + } + + public override void Flush() + { + } + + public override long Length + { + get { throw new NotSupportedException(); } + } + + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + if(mFinished) + throw new InvalidOperationException("CRC calculation has been finished."); + + mProcessed += count; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Update(mCRC, buffer, offset, count); + mTarget.Write(buffer, offset, count); + } + } + + class ReadingCrcBuilderStream: Stream + { + private long mProcessed; + private Stream mSource; + private uint mCRC; + private bool mFinished; + + public ReadingCrcBuilderStream(Stream source) + { + mSource = source; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.kInitCRC; + } + + protected override void Dispose(bool disposing) + { + try + { + if(disposing) + mSource.Dispose(); + } + finally + { + base.Dispose(disposing); + } + } + + public long Processed + { + get { return mProcessed; } + } + + public uint Finish() + { + if(!mFinished) + { + mFinished = true; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Finish(mCRC); + } + + return mCRC; + } + + public override bool CanRead + { + get { return mSource.CanRead; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return false; } + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override long Length + { + get { throw new NotImplementedException(); } + } + + public override long Position + { + get { throw new NotImplementedException(); } + set { throw new NotImplementedException(); } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if(count > 0 && !mFinished) + { + int read = mSource.Read(buffer, offset, count); + if(read > 0) + { + mProcessed += read; + mCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Update(mCRC, buffer, offset, read); + return read; + } + + Finish(); + } + + return 0; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/CrcCheckStream.cs b/SharpCompress/Compressor/LZMA/Utilites/CrcCheckStream.cs new file mode 100644 index 00000000..32ca0913 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/CrcCheckStream.cs @@ -0,0 +1,114 @@ +using System; +using System.IO; + +namespace SharpCompress.Compressor.LZMA.Utilites +{ + class CrcCheckStream: Stream + { + private readonly uint mExpectedCRC; + private uint mCurrentCRC; + private bool mClosed; + + private long[] mBytes = new long[256]; + private long mLength; + + public CrcCheckStream(uint crc) + { + mExpectedCRC = crc; + mCurrentCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.kInitCRC; + } + + protected override void Dispose(bool disposing) + { + if (mCurrentCRC != mExpectedCRC) + throw new InvalidOperationException(); + try + { + if(disposing && !mClosed) + { + mClosed = true; + mCurrentCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Finish(mCurrentCRC); +#if DEBUG + if(mCurrentCRC == mExpectedCRC) + System.Diagnostics.Debug.WriteLine("CRC ok: " + mExpectedCRC.ToString("x8")); + else + { + System.Diagnostics.Debugger.Break(); + System.Diagnostics.Debug.WriteLine("bad CRC"); + } + + double lengthInv = 1.0 / mLength; + double entropy = 0; + for(int i = 0; i < 256; i++) + { + if(mBytes[i] != 0) + { + double p = lengthInv * mBytes[i]; + entropy -= p * Math.Log(p, 256); + } + } + System.Diagnostics.Debug.WriteLine("entropy: " + (int)(entropy * 100) + "%"); +#endif + } + } + finally + { + base.Dispose(disposing); + } + } + + public override bool CanRead + { + get { return false; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return true; } + } + + public override void Flush() + { + } + + public override long Length + { + get { throw new NotSupportedException(); } + } + + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + mLength += count; + for(int i = 0; i < count; i++) + mBytes[buffer[offset + i]]++; + + mCurrentCRC = ManagedLzma.LZMA.Master.SevenZip.CRC.Update(mCurrentCRC, buffer, offset, count); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/IPasswordProvider.cs b/SharpCompress/Compressor/LZMA/Utilites/IPasswordProvider.cs new file mode 100644 index 00000000..83431c0a --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/IPasswordProvider.cs @@ -0,0 +1,7 @@ +namespace SharpCompress.Compressor.LZMA.Utilites +{ + public interface IPasswordProvider + { + string CryptoGetTextPassword(); + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/SyncStreamView.cs b/SharpCompress/Compressor/LZMA/Utilites/SyncStreamView.cs new file mode 100644 index 00000000..6400a202 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/SyncStreamView.cs @@ -0,0 +1,108 @@ +using System; +using System.IO; + +namespace SharpCompress.Compressor.LZMA.Utilites +{ + /// + /// Allows reading the same stream from multiple threads by synchronizing read access. + /// + class SyncStreamView: Stream + { + private object mSync; + private Stream mStream; + private long mOrigin; + private long mEnding; + private long mOffset; + + public SyncStreamView(object sync, Stream stream, long origin, long length) + { + mSync = sync; + mStream = stream; + mOrigin = origin; + mEnding = checked(origin + length); + mOffset = 0; + } + + public override bool CanRead + { + get { return true; } + } + + public override bool CanSeek + { + get { return true; } + } + + public override bool CanWrite + { + get { return false; } + } + + public override void Flush() + { + throw new InvalidOperationException(); + } + + public override long Length + { + get { return mEnding - mOrigin; } + } + + public override long Position + { + get { return mOffset; } + set + { + if(value < 0 || value > Length) + throw new ArgumentOutOfRangeException("value"); + + mOffset = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + long remaining = mEnding - mOrigin - mOffset; + if(count > remaining) + count = (int)remaining; + + if(count == 0) + return 0; + + int delta; + lock(mSync) + { + mStream.Position = mOrigin + mOffset; + delta = mStream.Read(buffer, offset, count); + } + + mOffset += delta; + return delta; + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch(origin) + { + case SeekOrigin.Begin: + return Position = offset; + case SeekOrigin.Current: + return Position += offset; + case SeekOrigin.End: + return Position = Length + offset; + default: + throw new ArgumentOutOfRangeException("origin"); + } + } + + public override void SetLength(long value) + { + throw new InvalidOperationException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException(); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/UnpackSubStream.cs b/SharpCompress/Compressor/LZMA/Utilites/UnpackSubStream.cs new file mode 100644 index 00000000..67547dcf --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/UnpackSubStream.cs @@ -0,0 +1,96 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace master._7zip.Utilities +{ + /// + /// This stream is a length-constrained wrapper around a cached stream so it does not dispose the inner stream. + /// + internal class UnpackSubStream: Stream + { + private Stream mSource; + private long mLength; + private long mOffset; + + internal UnpackSubStream(Stream source, long length) + { + mSource = source; + mLength = length; + } + + public override bool CanRead + { + get { return mSource.CanRead; } + } + + public override bool CanSeek + { + get { return false; } + } + + public override bool CanWrite + { + get { return false; } + } + + public override void Flush() + { + throw new NotSupportedException(); + } + + public override long Length + { + get { return mLength; } + } + + public override long Position + { + get { return mOffset; } + set { throw new NotSupportedException(); } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if(buffer == null) + throw new ArgumentNullException("buffer"); + + if(offset < 0 || offset > buffer.Length) + throw new ArgumentOutOfRangeException("offset"); + + if(count < 0 || count > buffer.Length - offset) + throw new ArgumentOutOfRangeException("count"); + + if(count > mLength - mOffset) + count = (int)(mLength - mOffset); + + if(count == 0) + return 0; + + int processed = mSource.Read(buffer, offset, count); + if(processed == 0) + throw new EndOfStreamException("Decoded stream ended prematurely, unpacked data is corrupt."); + + mOffset += processed; + return processed; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } +} diff --git a/SharpCompress/Compressor/LZMA/Utilites/Utils.cs b/SharpCompress/Compressor/LZMA/Utilites/Utils.cs new file mode 100644 index 00000000..0eabcb82 --- /dev/null +++ b/SharpCompress/Compressor/LZMA/Utilites/Utils.cs @@ -0,0 +1,80 @@ +using System; +using System.Diagnostics; +using System.IO; + +namespace SharpCompress.Compressor.LZMA.Utilites +{ + internal enum BlockType: byte + { + #region Constants + + End = 0, + Header = 1, + ArchiveProperties = 2, + AdditionalStreamsInfo = 3, + MainStreamsInfo = 4, + FilesInfo = 5, + PackInfo = 6, + UnpackInfo = 7, + SubStreamsInfo = 8, + Size = 9, + CRC = 10, + Folder = 11, + CodersUnpackSize = 12, + NumUnpackStream = 13, + EmptyStream = 14, + EmptyFile = 15, + Anti = 16, + Name = 17, + CTime = 18, + ATime = 19, + MTime = 20, + WinAttributes = 21, + Comment = 22, + EncodedHeader = 23, + StartPos = 24, + Dummy = 25, + + #endregion + } + + internal static class Utils + { + [Conditional("DEBUG")] + public static void Assert(bool expression) + { + if(!expression) + { + if(Debugger.IsAttached) + Debugger.Break(); + + throw new Exception("Assertion failed."); + } + } + + public static void ReadExact(this Stream stream, byte[] buffer, int offset, int length) + { + if(stream == null) + throw new ArgumentNullException("stream"); + + if(buffer == null) + throw new ArgumentNullException("buffer"); + + if(offset < 0 || offset > buffer.Length) + throw new ArgumentOutOfRangeException("offset"); + + if(length < 0 || length > buffer.Length - offset) + throw new ArgumentOutOfRangeException("length"); + + while(length > 0) + { + int fetched = stream.Read(buffer, offset, length); + if(fetched <= 0) + throw new EndOfStreamException(); + + offset += fetched; + length -= fetched; + } + } + } +} diff --git a/SharpCompress/SharpCompress.3.5.csproj b/SharpCompress/SharpCompress.3.5.csproj index ffb843f1..c034d158 100644 --- a/SharpCompress/SharpCompress.3.5.csproj +++ b/SharpCompress/SharpCompress.3.5.csproj @@ -127,15 +127,18 @@ - - - - + + + + + + + + + - - @@ -189,8 +192,14 @@ + + + + + + @@ -202,6 +211,13 @@ + + + + + + + diff --git a/SharpCompress/SharpCompress.Silverlight.csproj b/SharpCompress/SharpCompress.Silverlight.csproj index 5c6585f7..a806eecd 100644 --- a/SharpCompress/SharpCompress.Silverlight.csproj +++ b/SharpCompress/SharpCompress.Silverlight.csproj @@ -142,15 +142,18 @@ - - - - + + + + + + + + + - - @@ -204,8 +207,13 @@ + + + + + @@ -217,6 +225,13 @@ + + + + + + + diff --git a/SharpCompress/SharpCompress.WP7.csproj b/SharpCompress/SharpCompress.WP7.csproj index 4d7a74fa..d25a606f 100644 --- a/SharpCompress/SharpCompress.WP7.csproj +++ b/SharpCompress/SharpCompress.WP7.csproj @@ -114,15 +114,18 @@ - - - - + + + + + + + + + - - @@ -173,8 +176,13 @@ + + + + + @@ -186,6 +194,13 @@ + + + + + + + diff --git a/SharpCompress/SharpCompress.csproj b/SharpCompress/SharpCompress.csproj index f973b47b..fe59c9e9 100644 --- a/SharpCompress/SharpCompress.csproj +++ b/SharpCompress/SharpCompress.csproj @@ -50,7 +50,7 @@ full false ..\bin\ - DEBUG;TRACE + TRACE;DEBUG;DISABLE_TRACE prompt 4 true @@ -116,13 +116,6 @@ - - - - - - - @@ -133,7 +126,14 @@ - + + + + + + + + @@ -169,9 +169,48 @@ + + Code + + + + + + + + + + + + Code + + + + + + + + + + + + Code + + + Code + + + Code + + + Code + + + Code + @@ -218,16 +257,6 @@ - - - - - - - - - - @@ -342,6 +371,7 @@ +