starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-24 07:54:39 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

add perf testing project using JB memory and cpu

Browse Source
This commit is contained in:
Adam Hathcock
2025-10-23 09:39:57 +01:00
parent 4cca232d83
commit c108f2dcf3
7 changed files with 449 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Directory.Packages.props
View File

@@ -3,6 +3,7 @@
<PackageVersion Include="Bullseye" Version="6.0.0" />
<PackageVersion Include="AwesomeAssertions" Version="9.2.1" />
<PackageVersion Include="Glob" Version="1.1.9" />
<PackageVersion Include="JetBrains.Profiler.SelfApi" Version="2.5.14" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
<PackageVersion Include="Mono.Posix.NETStandard" Version="1.0.0" />

7
SharpCompress.sln
View File

@@ -23,6 +23,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Config", "Config", "{CDB425
.github\workflows\dotnetcore.yml = .github\workflows\dotnetcore.yml
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCompress.Performance", "tests\SharpCompress.Performance\SharpCompress.Performance.csproj", "{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -41,6 +43,10 @@ Global
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.Build.0 = Release|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -48,5 +54,6 @@ Global
GlobalSection(NestedProjects) = preSolution
{FD19DDD8-72B2-4024-8665-0D1F7A2AA998} = {3C5BE746-03E5-4895-9988-0B57F162F86C}
{F2B1A1EB-0FA6-40D0-8908-E13247C7226F} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529}
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529}
EndGlobalSection
EndGlobal

49
tests/SharpCompress.Performance/JetbrainsProfiler.cs Normal file
View File

@@ -0,0 +1,49 @@
using System;
using JetBrains.Profiler.SelfApi;
namespace SharpCompress.Test;
public static class JetbrainsProfiler
{
private sealed class CpuClass : IDisposable
{
public CpuClass(string snapshotPath)
{
DotTrace.Init();
var config2 = new DotTrace.Config();
config2.SaveToDir(snapshotPath);
DotTrace.Attach(config2);
DotTrace.StartCollectingData();
}
public void Dispose()
{
DotTrace.StopCollectingData();
DotTrace.SaveData();
DotTrace.Detach();
}
}
private sealed class MemoryClass : IDisposable
{
public MemoryClass(string snapshotPath)
{
DotMemory.Init();
var config = new DotMemory.Config();
config.UseLogLevelVerbose();
config.SaveToDir(snapshotPath);
DotMemory.Attach(config);
DotMemory.GetSnapshot("Before");
}
public void Dispose()
{
DotMemory.GetSnapshot("After");
DotMemory.Detach();
}
}
public static IDisposable Cpu(string snapshotPath) => new CpuClass(snapshotPath);
public static IDisposable Memory(string snapshotPath) => new MemoryClass(snapshotPath);
}

280
tests/SharpCompress.Performance/LargeMemoryStream.cs Normal file
View File

@@ -0,0 +1,280 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace SharpCompress.Performance;
/// <summary>
/// A Stream implementation backed by a List of byte arrays that supports large position values.
/// This allows handling streams larger than typical 32-bit or even standard 64-bit constraints
/// by chunking data into multiple byte array segments.
/// </summary>
public class LargeMemoryStream : Stream
{
private readonly List<byte[]> _chunks;
private readonly int _chunkSize;
private long _position;
private bool _isDisposed;
/// <summary>
/// Initializes a new instance of the LargePositionStream class.
/// </summary>
/// <param name="chunkSize">The size of each chunk in the backing byte array list. Defaults to 1MB.</param>
public LargeMemoryStream(int chunkSize = 1024 * 1024)
{
if (chunkSize <= 0)
throw new ArgumentException("Chunk size must be greater than zero.", nameof(chunkSize));
_chunks = new List<byte[]>();
_chunkSize = chunkSize;
_position = 0;
}
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => true;
public override long Length
{
get
{
ThrowIfDisposed();
if (_chunks.Count == 0)
return 0;
long length = (long)(_chunks.Count - 1) * _chunkSize;
length += _chunks[_chunks.Count - 1].Length;
return length;
}
}
public override long Position
{
get
{
ThrowIfDisposed();
return _position;
}
set
{
ThrowIfDisposed();
if (value < 0)
throw new ArgumentOutOfRangeException(
nameof(value),
"Position cannot be negative."
);
_position = value;
}
}
public override void Flush()
{
ThrowIfDisposed();
// No-op for in-memory stream
}
public override int Read(byte[] buffer, int offset, int count)
{
ThrowIfDisposed();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentOutOfRangeException();
long length = Length;
if (_position >= length)
return 0;
int bytesToRead = (int)Math.Min(count, length - _position);
int bytesRead = 0;
while (bytesRead < bytesToRead)
{
long chunkIndex = _position / _chunkSize;
int chunkOffset = (int)(_position % _chunkSize);
if (chunkIndex >= _chunks.Count)
break;
byte[] chunk = _chunks[(int)chunkIndex];
int availableInChunk = chunk.Length - chunkOffset;
int bytesToCopyFromChunk = Math.Min(availableInChunk, bytesToRead - bytesRead);
Array.Copy(chunk, chunkOffset, buffer, offset + bytesRead, bytesToCopyFromChunk);
_position += bytesToCopyFromChunk;
bytesRead += bytesToCopyFromChunk;
}
return bytesRead;
}
public override void Write(byte[] buffer, int offset, int count)
{
ThrowIfDisposed();
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
if (offset < 0 || count < 0 || offset + count > buffer.Length)
throw new ArgumentOutOfRangeException();
int bytesWritten = 0;
while (bytesWritten < count)
{
long chunkIndex = _position / _chunkSize;
int chunkOffset = (int)(_position % _chunkSize);
// Ensure we have enough chunks
while (_chunks.Count <= chunkIndex)
{
_chunks.Add(new byte[_chunkSize]);
}
byte[] chunk = _chunks[(int)chunkIndex];
int availableInChunk = chunk.Length - chunkOffset;
int bytesToCopyToChunk = Math.Min(availableInChunk, count - bytesWritten);
Array.Copy(buffer, offset + bytesWritten, chunk, chunkOffset, bytesToCopyToChunk);
_position += bytesToCopyToChunk;
bytesWritten += bytesToCopyToChunk;
}
}
public override long Seek(long offset, SeekOrigin origin)
{
ThrowIfDisposed();
long newPosition = origin switch
{
SeekOrigin.Begin => offset,
SeekOrigin.Current => _position + offset,
SeekOrigin.End => Length + offset,
_ => throw new ArgumentOutOfRangeException(nameof(origin)),
};
if (newPosition < 0)
throw new ArgumentOutOfRangeException(
nameof(offset),
"Cannot seek before the beginning of the stream."
);
_position = newPosition;
return _position;
}
public override void SetLength(long value)
{
ThrowIfDisposed();
if (value < 0)
throw new ArgumentOutOfRangeException(nameof(value), "Length cannot be negative.");
long currentLength = Length;
if (value < currentLength)
{
// Truncate
long chunkIndex = (value + _chunkSize - 1) / _chunkSize;
if (chunkIndex > 0)
chunkIndex--;
_chunks.RemoveRange((int)(chunkIndex + 1), _chunks.Count - (int)(chunkIndex + 1));
if (chunkIndex < _chunks.Count)
{
int lastChunkSize = (int)(value - chunkIndex * _chunkSize);
var x = _chunks[(int)chunkIndex];
Array.Resize(ref x, lastChunkSize);
}
if (_position > value)
_position = value;
}
else if (value > currentLength)
{
// Extend with zeros
long chunkIndex = currentLength / _chunkSize;
int chunkOffset = (int)(currentLength % _chunkSize);
while ((long)_chunks.Count * _chunkSize < value)
{
_chunks.Add(new byte[_chunkSize]);
}
// Resize the last chunk if needed
if (_chunks.Count > 0)
{
long lastChunkNeededSize = value - (long)(_chunks.Count - 1) * _chunkSize;
if (lastChunkNeededSize < _chunkSize)
{
var x = _chunks[^1];
Array.Resize(ref x, (int)lastChunkNeededSize);
}
}
}
}
/// <summary>
/// Gets the number of chunks in the backing list.
/// </summary>
public int ChunkCount => _chunks.Count;
/// <summary>
/// Gets the size of each chunk in bytes.
/// </summary>
public int ChunkSize => _chunkSize;
/// <summary>
/// Converts the stream contents to a single byte array.
/// This may consume significant memory for large streams.
/// </summary>
public byte[] ToArray()
{
ThrowIfDisposed();
long length = Length;
byte[] result = new byte[length];
long currentPosition = _position;
try
{
_position = 0;
int totalRead = 0;
while (totalRead < length)
{
int bytesToRead = (int)Math.Min(length - totalRead, int.MaxValue);
int bytesRead = Read(result, totalRead, bytesToRead);
if (bytesRead == 0)
break;
totalRead += bytesRead;
}
}
finally
{
_position = currentPosition;
}
return result;
}
private void ThrowIfDisposed()
{
if (_isDisposed)
throw new ObjectDisposedException(GetType().Name);
}
protected override void Dispose(bool disposing)
{
if (!_isDisposed)
{
if (disposing)
{
_chunks.Clear();
}
_isDisposed = true;
}
base.Dispose(disposing);
}
}

52
tests/SharpCompress.Performance/Program.cs Normal file
View File

@@ -0,0 +1,52 @@
// See https://aka.ms/new-console-template for more information
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Archives;
using SharpCompress.Performance;
using SharpCompress.Readers;
using SharpCompress.Test;
var index = AppDomain.CurrentDomain.BaseDirectory.IndexOf(
"SharpCompress.Performance",
StringComparison.OrdinalIgnoreCase
);
var path = AppDomain.CurrentDomain.BaseDirectory.Substring(0, index);
var SOLUTION_BASE_PATH = Path.GetDirectoryName(path) ?? throw new ArgumentNullException();
var TEST_ARCHIVES_PATH = Path.Combine(SOLUTION_BASE_PATH, "TestArchives", "Archives");
using var _ = JetbrainsProfiler.Memory($"/Users/adam/temp/");
//using var __ = JetbrainsProfiler.Cpu($"/Users/adam/temp/");
var testArchives = new[]
{
// "Rar.Audio_program.rar"
"64bitstream.zip.7z",
//"TarWithSymlink.tar.gz"
};
var arcs = testArchives.Select(a => Path.Combine(TEST_ARCHIVES_PATH, a)).ToArray();
for (int i = 0; i < 20; i++)
{
using var found = ArchiveFactory.Open(arcs[0]);
foreach (var entry in found.Entries.Where(entry => !entry.IsDirectory))
{
Console.WriteLine($"Extracting {entry.Key}");
using var entryStream = entry.OpenEntryStream();
entryStream.CopyTo(Stream.Null);
}
/*using var found = ReaderFactory.Open(arcs[0]);
while (found.MoveToNextEntry())
{
var entry = found.Entry;
if (entry.IsDirectory)
continue;
Console.WriteLine($"Extracting {entry.Key}");
found.WriteEntryTo(Stream.Null);
}*/
}
Console.WriteLine("Still running...");
await Task.Delay(5000);

10
tests/SharpCompress.Performance/SharpCompress.Performance.csproj Normal file
View File

@@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="JetBrains.Profiler.SelfApi" />
<ProjectReference Include="..\..\src\SharpCompress\SharpCompress.csproj" />
</ItemGroup>
</Project>

50
tests/SharpCompress.Performance/packages.lock.json Normal file
View File

@@ -0,0 +1,50 @@
{
"version": 2,
"dependencies": {
"net8.0": {
"JetBrains.Profiler.SelfApi": {
"type": "Direct",
"requested": "[2.5.14, )",
"resolved": "2.5.14",
"contentHash": "9+NcTe49B2M8/MOledSxKZkQKqavFf5xXZw4JL4bVu/KYiw6OOaD6cDQmNGSO18yUP/WoBXsXGKmZ9VOpmyadw==",
"dependencies": {
"JetBrains.HabitatDetector": "1.4.5",
"JetBrains.Profiler.Api": "1.4.10"
}
},
"JetBrains.FormatRipper": {
"type": "Transitive",
"resolved": "2.4.0",
"contentHash": "k5eGab1DArJH0k94ZO9oxDxg8go1KvR1oPGPzyVvfplEHetgrc2hGZ6Cken8fVsdS/Xp3hMnHd9L5MXb7JJM4A=="
},
"JetBrains.HabitatDetector": {
"type": "Transitive",
"resolved": "1.4.5",
"contentHash": "5kb1G32O8fmlS2QnJLycEnHbq9ukuDUHQll4mqOAPLEE1JEJcz12W6cTt1CMpQY3n/6R0jZAhmBvaJm2zixvow==",
"dependencies": {
"JetBrains.FormatRipper": "2.4.0"
}
},
"JetBrains.Profiler.Api": {
"type": "Transitive",
"resolved": "1.4.10",
"contentHash": "XBynPGDiWB6uWoiVwkki3uUsXqc66lRC1YX8LWYWc579ioJSB5OzZ8KsRK2q+eawj3OxrkeCsgXlb6mwBkCebQ==",
"dependencies": {
"JetBrains.HabitatDetector": "1.4.5"
}
},
"sharpcompress": {
"type": "Project",
"dependencies": {
"ZstdSharp.Port": "[0.8.6, )"
}
},
"ZstdSharp.Port": {
"type": "CentralTransitive",
"requested": "[0.8.6, )",
"resolved": "0.8.6",
"contentHash": "iP4jVLQoQmUjMU88g1WObiNr6YKZGvh4aOXn3yOJsHqZsflwRsxZPcIBvNXgjXO3vQKSLctXGLTpcBPLnWPS8A=="
}
}
}
}