diff --git a/Directory.Packages.props b/Directory.Packages.props index 4f095334..9a51cc35 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,7 @@ + diff --git a/SharpCompress.sln b/SharpCompress.sln index a86d1652..bbd08dae 100644 --- a/SharpCompress.sln +++ b/SharpCompress.sln @@ -23,6 +23,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Config", "Config", "{CDB425 .github\workflows\dotnetcore.yml = .github\workflows\dotnetcore.yml EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCompress.Performance", "tests\SharpCompress.Performance\SharpCompress.Performance.csproj", "{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -41,6 +43,10 @@ Global {D4D613CB-5E94-47FB-85BE-B8423D20C545}.Debug|Any CPU.Build.0 = Debug|Any CPU {D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.ActiveCfg = Release|Any CPU {D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.Build.0 = Release|Any CPU + {5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -48,5 +54,6 @@ Global GlobalSection(NestedProjects) = preSolution {FD19DDD8-72B2-4024-8665-0D1F7A2AA998} = {3C5BE746-03E5-4895-9988-0B57F162F86C} {F2B1A1EB-0FA6-40D0-8908-E13247C7226F} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529} + {5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529} EndGlobalSection EndGlobal diff --git a/tests/SharpCompress.Performance/JetbrainsProfiler.cs b/tests/SharpCompress.Performance/JetbrainsProfiler.cs new file mode 100644 index 00000000..9404fd81 --- /dev/null +++ b/tests/SharpCompress.Performance/JetbrainsProfiler.cs @@ -0,0 +1,49 @@ +using System; +using JetBrains.Profiler.SelfApi; + +namespace SharpCompress.Test; + +public static class JetbrainsProfiler +{ + private sealed class CpuClass : IDisposable + { + public CpuClass(string snapshotPath) + { + DotTrace.Init(); + var config2 = new DotTrace.Config(); + config2.SaveToDir(snapshotPath); + DotTrace.Attach(config2); + DotTrace.StartCollectingData(); + } + + public void Dispose() + { + DotTrace.StopCollectingData(); + DotTrace.SaveData(); + DotTrace.Detach(); + } + } + + private sealed class MemoryClass : IDisposable + { + public MemoryClass(string snapshotPath) + { + DotMemory.Init(); + var config = new DotMemory.Config(); + config.UseLogLevelVerbose(); + config.SaveToDir(snapshotPath); + DotMemory.Attach(config); + DotMemory.GetSnapshot("Before"); + } + + public void Dispose() + { + DotMemory.GetSnapshot("After"); + DotMemory.Detach(); + } + } + + public static IDisposable Cpu(string snapshotPath) => new CpuClass(snapshotPath); + + public static IDisposable Memory(string snapshotPath) => new MemoryClass(snapshotPath); +} diff --git a/tests/SharpCompress.Performance/LargeMemoryStream.cs b/tests/SharpCompress.Performance/LargeMemoryStream.cs new file mode 100644 index 00000000..ac0bcc25 --- /dev/null +++ b/tests/SharpCompress.Performance/LargeMemoryStream.cs @@ -0,0 +1,280 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace SharpCompress.Performance; + +/// +/// A Stream implementation backed by a List of byte arrays that supports large position values. +/// This allows handling streams larger than typical 32-bit or even standard 64-bit constraints +/// by chunking data into multiple byte array segments. +/// +public class LargeMemoryStream : Stream +{ + private readonly List _chunks; + private readonly int _chunkSize; + private long _position; + private bool _isDisposed; + + /// + /// Initializes a new instance of the LargePositionStream class. + /// + /// The size of each chunk in the backing byte array list. Defaults to 1MB. + public LargeMemoryStream(int chunkSize = 1024 * 1024) + { + if (chunkSize <= 0) + throw new ArgumentException("Chunk size must be greater than zero.", nameof(chunkSize)); + + _chunks = new List(); + _chunkSize = chunkSize; + _position = 0; + } + + public override bool CanRead => true; + + public override bool CanSeek => true; + + public override bool CanWrite => true; + + public override long Length + { + get + { + ThrowIfDisposed(); + if (_chunks.Count == 0) + return 0; + + long length = (long)(_chunks.Count - 1) * _chunkSize; + length += _chunks[_chunks.Count - 1].Length; + return length; + } + } + + public override long Position + { + get + { + ThrowIfDisposed(); + return _position; + } + set + { + ThrowIfDisposed(); + if (value < 0) + throw new ArgumentOutOfRangeException( + nameof(value), + "Position cannot be negative." + ); + _position = value; + } + } + + public override void Flush() + { + ThrowIfDisposed(); + // No-op for in-memory stream + } + + public override int Read(byte[] buffer, int offset, int count) + { + ThrowIfDisposed(); + if (buffer == null) + throw new ArgumentNullException(nameof(buffer)); + if (offset < 0 || count < 0 || offset + count > buffer.Length) + throw new ArgumentOutOfRangeException(); + + long length = Length; + if (_position >= length) + return 0; + + int bytesToRead = (int)Math.Min(count, length - _position); + int bytesRead = 0; + + while (bytesRead < bytesToRead) + { + long chunkIndex = _position / _chunkSize; + int chunkOffset = (int)(_position % _chunkSize); + + if (chunkIndex >= _chunks.Count) + break; + + byte[] chunk = _chunks[(int)chunkIndex]; + int availableInChunk = chunk.Length - chunkOffset; + int bytesToCopyFromChunk = Math.Min(availableInChunk, bytesToRead - bytesRead); + + Array.Copy(chunk, chunkOffset, buffer, offset + bytesRead, bytesToCopyFromChunk); + + _position += bytesToCopyFromChunk; + bytesRead += bytesToCopyFromChunk; + } + + return bytesRead; + } + + public override void Write(byte[] buffer, int offset, int count) + { + ThrowIfDisposed(); + if (buffer == null) + throw new ArgumentNullException(nameof(buffer)); + if (offset < 0 || count < 0 || offset + count > buffer.Length) + throw new ArgumentOutOfRangeException(); + + int bytesWritten = 0; + + while (bytesWritten < count) + { + long chunkIndex = _position / _chunkSize; + int chunkOffset = (int)(_position % _chunkSize); + + // Ensure we have enough chunks + while (_chunks.Count <= chunkIndex) + { + _chunks.Add(new byte[_chunkSize]); + } + + byte[] chunk = _chunks[(int)chunkIndex]; + int availableInChunk = chunk.Length - chunkOffset; + int bytesToCopyToChunk = Math.Min(availableInChunk, count - bytesWritten); + + Array.Copy(buffer, offset + bytesWritten, chunk, chunkOffset, bytesToCopyToChunk); + + _position += bytesToCopyToChunk; + bytesWritten += bytesToCopyToChunk; + } + } + + public override long Seek(long offset, SeekOrigin origin) + { + ThrowIfDisposed(); + + long newPosition = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + + if (newPosition < 0) + throw new ArgumentOutOfRangeException( + nameof(offset), + "Cannot seek before the beginning of the stream." + ); + + _position = newPosition; + return _position; + } + + public override void SetLength(long value) + { + ThrowIfDisposed(); + if (value < 0) + throw new ArgumentOutOfRangeException(nameof(value), "Length cannot be negative."); + + long currentLength = Length; + + if (value < currentLength) + { + // Truncate + long chunkIndex = (value + _chunkSize - 1) / _chunkSize; + if (chunkIndex > 0) + chunkIndex--; + + _chunks.RemoveRange((int)(chunkIndex + 1), _chunks.Count - (int)(chunkIndex + 1)); + + if (chunkIndex < _chunks.Count) + { + int lastChunkSize = (int)(value - chunkIndex * _chunkSize); + var x = _chunks[(int)chunkIndex]; + Array.Resize(ref x, lastChunkSize); + } + + if (_position > value) + _position = value; + } + else if (value > currentLength) + { + // Extend with zeros + long chunkIndex = currentLength / _chunkSize; + int chunkOffset = (int)(currentLength % _chunkSize); + + while ((long)_chunks.Count * _chunkSize < value) + { + _chunks.Add(new byte[_chunkSize]); + } + + // Resize the last chunk if needed + if (_chunks.Count > 0) + { + long lastChunkNeededSize = value - (long)(_chunks.Count - 1) * _chunkSize; + if (lastChunkNeededSize < _chunkSize) + { + var x = _chunks[^1]; + Array.Resize(ref x, (int)lastChunkNeededSize); + } + } + } + } + + /// + /// Gets the number of chunks in the backing list. + /// + public int ChunkCount => _chunks.Count; + + /// + /// Gets the size of each chunk in bytes. + /// + public int ChunkSize => _chunkSize; + + /// + /// Converts the stream contents to a single byte array. + /// This may consume significant memory for large streams. + /// + public byte[] ToArray() + { + ThrowIfDisposed(); + long length = Length; + byte[] result = new byte[length]; + long currentPosition = _position; + + try + { + _position = 0; + int totalRead = 0; + while (totalRead < length) + { + int bytesToRead = (int)Math.Min(length - totalRead, int.MaxValue); + int bytesRead = Read(result, totalRead, bytesToRead); + if (bytesRead == 0) + break; + totalRead += bytesRead; + } + } + finally + { + _position = currentPosition; + } + + return result; + } + + private void ThrowIfDisposed() + { + if (_isDisposed) + throw new ObjectDisposedException(GetType().Name); + } + + protected override void Dispose(bool disposing) + { + if (!_isDisposed) + { + if (disposing) + { + _chunks.Clear(); + } + _isDisposed = true; + } + + base.Dispose(disposing); + } +} diff --git a/tests/SharpCompress.Performance/Program.cs b/tests/SharpCompress.Performance/Program.cs new file mode 100644 index 00000000..c5a89165 --- /dev/null +++ b/tests/SharpCompress.Performance/Program.cs @@ -0,0 +1,52 @@ +// See https://aka.ms/new-console-template for more information +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Performance; +using SharpCompress.Readers; +using SharpCompress.Test; + +var index = AppDomain.CurrentDomain.BaseDirectory.IndexOf( + "SharpCompress.Performance", + StringComparison.OrdinalIgnoreCase +); +var path = AppDomain.CurrentDomain.BaseDirectory.Substring(0, index); +var SOLUTION_BASE_PATH = Path.GetDirectoryName(path) ?? throw new ArgumentNullException(); + +var TEST_ARCHIVES_PATH = Path.Combine(SOLUTION_BASE_PATH, "TestArchives", "Archives"); +using var _ = JetbrainsProfiler.Memory($"/Users/adam/temp/"); + +//using var __ = JetbrainsProfiler.Cpu($"/Users/adam/temp/"); +var testArchives = new[] +{ + // "Rar.Audio_program.rar" + "64bitstream.zip.7z", + //"TarWithSymlink.tar.gz" +}; +var arcs = testArchives.Select(a => Path.Combine(TEST_ARCHIVES_PATH, a)).ToArray(); + +for (int i = 0; i < 20; i++) +{ + using var found = ArchiveFactory.Open(arcs[0]); + foreach (var entry in found.Entries.Where(entry => !entry.IsDirectory)) + { + Console.WriteLine($"Extracting {entry.Key}"); + using var entryStream = entry.OpenEntryStream(); + entryStream.CopyTo(Stream.Null); + } + /*using var found = ReaderFactory.Open(arcs[0]); + while (found.MoveToNextEntry()) + { + var entry = found.Entry; + if (entry.IsDirectory) + continue; + + Console.WriteLine($"Extracting {entry.Key}"); + found.WriteEntryTo(Stream.Null); + }*/ +} + +Console.WriteLine("Still running..."); +await Task.Delay(5000); diff --git a/tests/SharpCompress.Performance/SharpCompress.Performance.csproj b/tests/SharpCompress.Performance/SharpCompress.Performance.csproj new file mode 100644 index 00000000..460b1d53 --- /dev/null +++ b/tests/SharpCompress.Performance/SharpCompress.Performance.csproj @@ -0,0 +1,10 @@ + + + Exe + net8.0 + + + + + + diff --git a/tests/SharpCompress.Performance/packages.lock.json b/tests/SharpCompress.Performance/packages.lock.json new file mode 100644 index 00000000..5c6a2b9e --- /dev/null +++ b/tests/SharpCompress.Performance/packages.lock.json @@ -0,0 +1,50 @@ +{ + "version": 2, + "dependencies": { + "net8.0": { + "JetBrains.Profiler.SelfApi": { + "type": "Direct", + "requested": "[2.5.14, )", + "resolved": "2.5.14", + "contentHash": "9+NcTe49B2M8/MOledSxKZkQKqavFf5xXZw4JL4bVu/KYiw6OOaD6cDQmNGSO18yUP/WoBXsXGKmZ9VOpmyadw==", + "dependencies": { + "JetBrains.HabitatDetector": "1.4.5", + "JetBrains.Profiler.Api": "1.4.10" + } + }, + "JetBrains.FormatRipper": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "k5eGab1DArJH0k94ZO9oxDxg8go1KvR1oPGPzyVvfplEHetgrc2hGZ6Cken8fVsdS/Xp3hMnHd9L5MXb7JJM4A==" + }, + "JetBrains.HabitatDetector": { + "type": "Transitive", + "resolved": "1.4.5", + "contentHash": "5kb1G32O8fmlS2QnJLycEnHbq9ukuDUHQll4mqOAPLEE1JEJcz12W6cTt1CMpQY3n/6R0jZAhmBvaJm2zixvow==", + "dependencies": { + "JetBrains.FormatRipper": "2.4.0" + } + }, + "JetBrains.Profiler.Api": { + "type": "Transitive", + "resolved": "1.4.10", + "contentHash": "XBynPGDiWB6uWoiVwkki3uUsXqc66lRC1YX8LWYWc579ioJSB5OzZ8KsRK2q+eawj3OxrkeCsgXlb6mwBkCebQ==", + "dependencies": { + "JetBrains.HabitatDetector": "1.4.5" + } + }, + "sharpcompress": { + "type": "Project", + "dependencies": { + "ZstdSharp.Port": "[0.8.6, )" + } + }, + "ZstdSharp.Port": { + "type": "CentralTransitive", + "requested": "[0.8.6, )", + "resolved": "0.8.6", + "contentHash": "iP4jVLQoQmUjMU88g1WObiNr6YKZGvh4aOXn3yOJsHqZsflwRsxZPcIBvNXgjXO3vQKSLctXGLTpcBPLnWPS8A==" + } + } + } +} \ No newline at end of file