Compare commits

..

66 Commits

Author SHA1 Message Date
Adam Hathcock
7a476b98fb fmt 2025-10-23 15:31:31 +01:00
Adam Hathcock
ba089fb6cc fix some tests, rar fails 2025-10-23 15:31:18 +01:00
Adam Hathcock
c10cfa2a22 fmt 2025-10-23 15:04:44 +01:00
Adam Hathcock
1fb6ad4474 changed some tests 2025-10-23 15:04:31 +01:00
Adam Hathcock
d5f93c5c08 merge fixes 2025-10-23 14:48:24 +01:00
Adam Hathcock
773e3ac048 Merge remote-tracking branch 'origin/master' into adam/async-rar
# Conflicts:
#	src/SharpCompress/Archives/IArchiveEntryExtensions.cs
#	src/SharpCompress/Archives/IArchiveExtensions.cs
#	src/SharpCompress/Utility.cs
2025-10-23 14:45:36 +01:00
Adam Hathcock
e043e06656 Merge pull request #969 from adamhathcock/adam/perf
Add JB perf testing project.
2025-10-23 14:34:43 +01:00
Adam Hathcock
14b52599f4 Update src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-23 14:20:54 +01:00
Adam Hathcock
e3e2c0c567 Update tests/SharpCompress.Performance/LargeMemoryStream.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-23 14:19:16 +01:00
Adam Hathcock
4fc5d60f03 reduce visibility 2025-10-23 14:16:39 +01:00
Adam Hathcock
c37a9e0f82 Merge remote-tracking branch 'origin/adam/perf' into adam/perf 2025-10-23 13:50:31 +01:00
Adam Hathcock
fed17ebb96 fmt 2025-10-23 13:50:07 +01:00
Adam Hathcock
eeac678872 More usage of pool and better copy 2025-10-23 13:49:54 +01:00
Adam Hathcock
f9ed0f2df9 Update tests/SharpCompress.Performance/Program.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-23 11:47:42 +01:00
Adam Hathcock
0ddbacac85 Update src/SharpCompress/Compressors/Rar/UnpackV1/UnpackUtility.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-23 11:47:27 +01:00
Adam Hathcock
f0d28aa5cf fmt 2025-10-23 11:43:38 +01:00
Adam Hathcock
cc84f6fee4 more making rar faster 2025-10-23 11:43:21 +01:00
Adam Hathcock
00e6eef369 used AI to optimize some copys and shifting 2025-10-23 11:18:50 +01:00
Adam Hathcock
1ae71907bc don't need to clear everything 2025-10-23 10:53:54 +01:00
Adam Hathcock
3ff688fba2 clear and null check 2025-10-23 10:48:18 +01:00
Adam Hathcock
bb59b3d456 add pool to LZMA out window 2025-10-23 09:54:52 +01:00
Adam Hathcock
186ea74ada add some fixes for rar to pool memory 2025-10-23 09:40:15 +01:00
Adam Hathcock
c108f2dcf3 add perf testing project using JB memory and cpu 2025-10-23 09:39:57 +01:00
Adam Hathcock
4cca232d83 Merge pull request #959 from adamhathcock/adam/xz-wrapped-often
Removed wrappers that weren't needed (probably)
2025-10-22 11:54:47 +01:00
Adam Hathcock
1db511e9cb Merge branch 'master' into adam/xz-wrapped-often 2025-10-22 11:51:46 +01:00
Adam Hathcock
76afa7d3bf Merge pull request #968 from adamhathcock/adam/rework-deps
rework dependencies to be correct for frameworks and update
2025-10-22 11:51:30 +01:00
Adam Hathcock
3513f7b1cd Update src/SharpCompress/SharpCompress.csproj
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-22 10:51:12 +01:00
Adam Hathcock
4531fe39e6 Merge branch 'master' into adam/rework-deps 2025-10-22 10:48:16 +01:00
Adam Hathcock
8d276a85bc rework dependencies to be correct for frameworks and update 2025-10-22 10:47:43 +01:00
Adam Hathcock
5f0d042bc3 Merge pull request #967 from adamhathcock/adam/reduce-custom-utilities
Reduce custom utilities for arrays/bytes
2025-10-22 10:41:10 +01:00
Adam Hathcock
408f07e3c4 Merge branch 'master' into adam/reduce-custom-utilities 2025-10-22 10:38:01 +01:00
Adam Hathcock
d1a540c90c use windows instead of skippable fact 2025-10-22 10:32:47 +01:00
Adam Hathcock
00df8e930e add windows only compile constant 2025-10-22 10:30:40 +01:00
Adam Hathcock
3b768b1b77 Merge pull request #961 from adamhathcock/dependabot/nuget/AwesomeAssertions-9.2.1
Bump AwesomeAssertions from 9.2.0 to 9.2.1
2025-10-22 10:25:01 +01:00
Adam Hathcock
42a7ececa0 Merge branch 'master' into adam/xz-wrapped-often 2025-10-22 10:22:36 +01:00
Adam Hathcock
e8867de049 Merge branch 'master' into dependabot/nuget/AwesomeAssertions-9.2.1 2025-10-22 10:21:59 +01:00
Adam Hathcock
a1dfa3dfa3 xplat tests for path characters 2025-10-22 10:21:22 +01:00
Adam Hathcock
83917d4f79 Merge remote-tracking branch 'origin/master' into adam/reduce-custom-utilities 2025-10-22 10:17:20 +01:00
Adam Hathcock
513cd4f905 some AI suggestions 2025-10-22 10:16:45 +01:00
Adam Hathcock
eda0309df3 Merge pull request #966 from adamhathcock/adam/reduce-stackalloc
Remove a dynamically created stackalloc
2025-10-22 10:13:14 +01:00
Adam Hathcock
74e27c028e fix the span length 2025-10-22 10:10:07 +01:00
Adam Hathcock
36c06c4089 ugh, this is used because it shadows a field 2025-10-22 09:32:19 +01:00
Adam Hathcock
249b8a9cdd add AI generated tests 2025-10-22 09:28:07 +01:00
Adam Hathcock
62bee15f00 fmt 2025-10-22 09:19:30 +01:00
Adam Hathcock
d8797b69e4 remove do while 2025-10-22 09:19:09 +01:00
Adam Hathcock
084fe72b02 Consolidate not null 2025-10-22 09:17:13 +01:00
Adam Hathcock
c823acaa3f optimize ReadFully and Skip 2025-10-22 09:10:16 +01:00
Adam Hathcock
e0d6cd9cb7 Try to reduce custom functions for array/byte management 2025-10-22 09:00:21 +01:00
Adam Hathcock
01021e102b remove some extra stackallocs 2025-10-22 08:36:03 +01:00
Adam Hathcock
6de738ff17 reduce dynamic stackallocs in unpackv1 2025-10-22 08:32:19 +01:00
Adam Hathcock
c0612547eb Merge pull request #964 from adamhathcock/adam/extract-all-solid-only
Only allow extract all on archives that are solid (some rars and 7zip only)
2025-10-21 14:08:23 +01:00
Adam Hathcock
e960907698 Update src/SharpCompress/Archives/AbstractArchive.cs
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-10-21 13:55:56 +01:00
Adam Hathcock
84e03b1b27 Allow 7zip files of all sizes? 2025-10-21 10:28:58 +01:00
Adam Hathcock
f1a80da34b fix tests that use extract all wrongly 2025-10-21 09:56:29 +01:00
Adam Hathcock
5a5a55e556 fmt 2025-10-21 09:22:35 +01:00
Adam Hathcock
e1f132b45b Only allow extract all on archives that are solid (some rars and 7zip only) 2025-10-21 09:21:46 +01:00
dependabot[bot]
087011aede Bump AwesomeAssertions from 9.2.0 to 9.2.1
---
updated-dependencies:
- dependency-name: AwesomeAssertions
  dependency-version: 9.2.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-10-20 10:44:17 +00:00
Adam Hathcock
d1409d6dde more methods for async 2025-10-17 11:01:37 +01:00
Adam Hathcock
e30a88e634 Started rar async...interface changes were viral 2025-10-17 10:50:09 +01:00
Adam Hathcock
1430bf9b31 fmt 2025-10-15 09:54:13 +01:00
Adam Hathcock
4e5de817ef Removed too many wrappers
# Conflicts:
#	src/SharpCompress/Compressors/Xz/XZIndex.cs
2025-10-15 09:53:46 +01:00
Adam Hathcock
5d6b94f8c3 Merge pull request #952 from adamhathcock/dependabot/github_actions/actions/checkout-5
Bump actions/checkout from 4 to 5
2025-10-14 08:25:53 +01:00
Adam Hathcock
8dfbe56f42 Merge branch 'master' into dependabot/github_actions/actions/checkout-5 2025-10-14 08:23:18 +01:00
Adam Hathcock
df79d983d7 Merge pull request #957 from adamhathcock/dependabot/github_actions/actions/setup-dotnet-5
Bump actions/setup-dotnet from 4 to 5
2025-10-14 08:22:47 +01:00
dependabot[bot]
6c23a28826 Bump actions/setup-dotnet from 4 to 5
Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4 to 5.
- [Release notes](https://github.com/actions/setup-dotnet/releases)
- [Commits](https://github.com/actions/setup-dotnet/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/setup-dotnet
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-10-13 16:21:25 +00:00
dependabot[bot]
f72289570a Bump actions/checkout from 4 to 5
Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-10-13 16:12:51 +00:00
332 changed files with 6618 additions and 60307 deletions

View File

@@ -14,8 +14,8 @@ jobs:
os: [windows-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-dotnet@v4
- uses: actions/checkout@v5
- uses: actions/setup-dotnet@v5
with:
dotnet-version: 8.0.x
- run: dotnet run --project build/build.csproj

View File

@@ -1,10 +1,11 @@
<Project>
<ItemGroup>
<PackageVersion Include="Bullseye" Version="6.0.0" />
<PackageVersion Include="AwesomeAssertions" Version="9.2.0" />
<PackageVersion Include="AwesomeAssertions" Version="9.2.1" />
<PackageVersion Include="Glob" Version="1.1.9" />
<PackageVersion Include="JetBrains.Profiler.SelfApi" Version="2.5.14" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
<PackageVersion Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageVersion Include="SimpleExec" Version="12.0.0" />
<PackageVersion Include="System.Buffers" Version="4.6.1" />
@@ -12,7 +13,6 @@
<PackageVersion Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageVersion Include="xunit" Version="2.9.3" />
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
<PackageVersion Include="xunit.SkippableFact" Version="1.5.23" />
<PackageVersion Include="ZstdSharp.Port" Version="0.8.6" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageVersion Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" />

View File

@@ -23,6 +23,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Config", "Config", "{CDB425
.github\workflows\dotnetcore.yml = .github\workflows\dotnetcore.yml
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCompress.Performance", "tests\SharpCompress.Performance\SharpCompress.Performance.csproj", "{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -41,6 +43,10 @@ Global
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D4D613CB-5E94-47FB-85BE-B8423D20C545}.Release|Any CPU.Build.0 = Release|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -48,5 +54,6 @@ Global
GlobalSection(NestedProjects) = preSolution
{FD19DDD8-72B2-4024-8665-0D1F7A2AA998} = {3C5BE746-03E5-4895-9988-0B57F162F86C}
{F2B1A1EB-0FA6-40D0-8908-E13247C7226F} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529}
{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17} = {0F0901FF-E8D9-426A-B5A2-17C7F47C1529}
EndGlobalSection
EndGlobal

View File

@@ -144,6 +144,12 @@ public abstract class AbstractArchive<TEntry, TVolume> : IArchive, IArchiveExtra
/// <returns></returns>
public IReader ExtractAllEntries()
{
if (!IsSolid && Type != ArchiveType.SevenZip)
{
throw new InvalidOperationException(
"ExtractAllEntries can only be used on solid archives or 7Zip archives (which require random access)."
);
}
((IArchiveExtractionListener)this).EnsureEntriesLoaded();
return CreateReaderForSolidExtraction();
}

View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Factories;
using SharpCompress.IO;
@@ -45,7 +46,7 @@ public static class ArchiveFactory
/// <param name="options"></param>
public static IArchive Open(string filePath, ReaderOptions? options = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
return Open(new FileInfo(filePath), options);
}
@@ -68,7 +69,7 @@ public static class ArchiveFactory
/// <param name="options"></param>
public static IArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? options = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var filesArray = fileInfos.ToArray();
if (filesArray.Length == 0)
{
@@ -81,7 +82,7 @@ public static class ArchiveFactory
return Open(fileInfo, options);
}
fileInfo.CheckNotNull(nameof(fileInfo));
fileInfo.NotNull(nameof(fileInfo));
options ??= new ReaderOptions { LeaveStreamOpen = false };
return FindFactory<IMultiArchiveFactory>(fileInfo).Open(filesArray, options);
@@ -94,7 +95,7 @@ public static class ArchiveFactory
/// <param name="options"></param>
public static IArchive Open(IEnumerable<Stream> streams, ReaderOptions? options = null)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var streamsArray = streams.ToArray();
if (streamsArray.Length == 0)
{
@@ -107,7 +108,7 @@ public static class ArchiveFactory
return Open(firstStream, options);
}
firstStream.CheckNotNull(nameof(firstStream));
firstStream.NotNull(nameof(firstStream));
options ??= new ReaderOptions();
return FindFactory<IMultiArchiveFactory>(firstStream).Open(streamsArray, options);
@@ -116,20 +117,20 @@ public static class ArchiveFactory
/// <summary>
/// Extract to specific directory, retaining filename
/// </summary>
public static void WriteToDirectory(
public static async Task WriteToDirectoryAsync(
string sourceArchive,
string destinationDirectory,
ExtractionOptions? options = null
)
{
using var archive = Open(sourceArchive);
archive.WriteToDirectory(destinationDirectory, options);
await archive.WriteToDirectoryAsync(destinationDirectory, options);
}
private static T FindFactory<T>(FileInfo finfo)
where T : IFactory
{
finfo.CheckNotNull(nameof(finfo));
finfo.NotNull(nameof(finfo));
using Stream stream = finfo.OpenRead();
return FindFactory<T>(stream);
}
@@ -137,7 +138,7 @@ public static class ArchiveFactory
private static T FindFactory<T>(Stream stream)
where T : IFactory
{
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (!stream.CanRead || !stream.CanSeek)
{
throw new ArgumentException("Stream should be readable and seekable");
@@ -172,7 +173,7 @@ public static class ArchiveFactory
int bufferSize = ReaderOptions.DefaultBufferSize
)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
using Stream s = File.OpenRead(filePath);
return IsArchive(s, out type, bufferSize);
}
@@ -184,7 +185,7 @@ public static class ArchiveFactory
)
{
type = null;
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (!stream.CanRead || !stream.CanSeek)
{
@@ -215,7 +216,7 @@ public static class ArchiveFactory
/// <returns></returns>
public static IEnumerable<string> GetFileParts(string part1)
{
part1.CheckNotNullOrEmpty(nameof(part1));
part1.NotNullOrEmpty(nameof(part1));
return GetFileParts(new FileInfo(part1)).Select(a => a.FullName);
}
@@ -226,7 +227,7 @@ public static class ArchiveFactory
/// <returns></returns>
public static IEnumerable<FileInfo> GetFileParts(FileInfo part1)
{
part1.CheckNotNull(nameof(part1));
part1.NotNull(nameof(part1));
yield return part1;
foreach (var factory in Factory.Factories.OfType<IFactory>())

View File

@@ -21,7 +21,7 @@ public class GZipArchive : AbstractWritableArchive<GZipArchiveEntry, GZipVolume>
/// <param name="readerOptions"></param>
public static GZipArchive Open(string filePath, ReaderOptions? readerOptions = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
return Open(new FileInfo(filePath), readerOptions ?? new ReaderOptions());
}
@@ -32,7 +32,7 @@ public class GZipArchive : AbstractWritableArchive<GZipArchiveEntry, GZipVolume>
/// <param name="readerOptions"></param>
public static GZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
fileInfo.NotNull(nameof(fileInfo));
return new GZipArchive(
new SourceStream(
fileInfo,
@@ -52,7 +52,7 @@ public class GZipArchive : AbstractWritableArchive<GZipArchiveEntry, GZipVolume>
ReaderOptions? readerOptions = null
)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var files = fileInfos.ToArray();
return new GZipArchive(
new SourceStream(
@@ -70,7 +70,7 @@ public class GZipArchive : AbstractWritableArchive<GZipArchiveEntry, GZipVolume>
/// <param name="readerOptions"></param>
public static GZipArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var strms = streams.ToArray();
return new GZipArchive(
new SourceStream(
@@ -88,7 +88,7 @@ public class GZipArchive : AbstractWritableArchive<GZipArchiveEntry, GZipVolume>
/// <param name="readerOptions"></param>
public static GZipArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (stream is not { CanSeek: true })
{

View File

@@ -1,5 +1,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common.GZip;
namespace SharpCompress.Archives.GZip;
@@ -20,6 +21,9 @@ public class GZipArchiveEntry : GZipEntry, IArchiveEntry
return Parts.Single().GetCompressedStream().NotNull();
}
public virtual async Task<Stream> OpenEntryStreamAsync() =>
await Task.FromResult(OpenEntryStream());
#region IArchiveEntry Members
public IArchive Archive { get; }

View File

@@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common;
namespace SharpCompress.Archives;
@@ -11,6 +12,12 @@ public interface IArchiveEntry : IEntry
/// </summary>
Stream OpenEntryStream();
/// <summary>
/// Opens the current entry as a stream that will decompress as it is read.
/// Read the entire stream or use SkipEntry on EntryStream.
/// </summary>
Task<Stream> OpenEntryStreamAsync();
/// <summary>
/// The archive can find all the parts of the archive needed to extract this entry.
/// </summary>

View File

@@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.IO;
@@ -6,7 +7,7 @@ namespace SharpCompress.Archives;
public static class IArchiveEntryExtensions
{
public static void WriteTo(this IArchiveEntry archiveEntry, Stream streamToWriteTo)
public static async Task WriteToAsync(this IArchiveEntry archiveEntry, Stream streamToWriteTo)
{
if (archiveEntry.IsDirectory)
{
@@ -21,11 +22,11 @@ public static class IArchiveEntryExtensions
archiveEntry.Size,
archiveEntry.CompressedSize
);
var entryStream = archiveEntry.OpenEntryStream();
var entryStream = await archiveEntry.OpenEntryStreamAsync();
using (entryStream)
{
using Stream s = new ListeningStream(streamListener, entryStream);
s.TransferTo(streamToWriteTo);
await s.CopyToAsync(streamToWriteTo);
}
streamListener.FireEntryExtractionEnd(archiveEntry);
}
@@ -33,34 +34,34 @@ public static class IArchiveEntryExtensions
/// <summary>
/// Extract to specific directory, retaining filename
/// </summary>
public static void WriteToDirectory(
public static async Task WriteEntryToDirectoryAsync(
this IArchiveEntry entry,
string destinationDirectory,
ExtractionOptions? options = null
) =>
ExtractionMethods.WriteEntryToDirectory(
await ExtractionMethods.WriteEntryToDirectoryAsync(
entry,
destinationDirectory,
options,
entry.WriteToFile
entry.WriteToFileAsync
);
/// <summary>
/// Extract to specific file
/// </summary>
public static void WriteToFile(
public static Task WriteToFileAsync(
this IArchiveEntry entry,
string destinationFileName,
ExtractionOptions? options = null
) =>
ExtractionMethods.WriteEntryToFile(
ExtractionMethods.WriteEntryToFileAsync(
entry,
destinationFileName,
options,
(x, fm) =>
async (x, fm) =>
{
using var fs = File.Open(destinationFileName, fm);
entry.WriteTo(fs);
await entry.WriteToAsync(fs);
}
);
}

View File

@@ -1,8 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Readers;
@@ -13,14 +13,14 @@ public static class IArchiveExtensions
/// <summary>
/// Extract to specific directory, retaining filename
/// </summary>
public static void WriteToDirectory(
public static async Task WriteToDirectoryAsync(
this IArchive archive,
string destinationDirectory,
ExtractionOptions? options = null
)
{
using var reader = archive.ExtractAllEntries();
reader.WriteAllToDirectory(destinationDirectory, options);
await reader.WriteAllToDirectoryAsync(destinationDirectory, options);
}
/// <summary>
@@ -30,7 +30,7 @@ public static class IArchiveExtensions
/// <param name="destination">The folder to extract into.</param>
/// <param name="progressReport">Optional progress report callback.</param>
/// <param name="cancellationToken">Optional cancellation token.</param>
public static void ExtractToDirectory(
public static async Task ExtractToDirectoryAsync(
this IArchive archive,
string destination,
Action<double>? progressReport = null,
@@ -45,12 +45,10 @@ public static class IArchiveExtensions
var seenDirectories = new HashSet<string>();
// Extract
var entries = archive.ExtractAllEntries();
while (entries.MoveToNextEntry())
foreach (var entry in archive.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
var entry = entries.Entry;
if (entry.IsDirectory)
{
var dirPath = Path.Combine(destination, entry.Key.NotNull("Entry Key is null"));
@@ -77,7 +75,7 @@ public static class IArchiveExtensions
// Write file
using var fs = File.OpenWrite(path);
entries.WriteEntryTo(fs);
await entry.WriteToAsync(fs);
// Update progress
bytesRead += entry.Size;

View File

@@ -95,7 +95,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
/// <param name="options"></param>
public static RarArchive Open(string filePath, ReaderOptions? options = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
var fileInfo = new FileInfo(filePath);
return new RarArchive(
new SourceStream(
@@ -113,7 +113,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
/// <param name="options"></param>
public static RarArchive Open(FileInfo fileInfo, ReaderOptions? options = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
fileInfo.NotNull(nameof(fileInfo));
return new RarArchive(
new SourceStream(
fileInfo,
@@ -130,7 +130,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
/// <param name="options"></param>
public static RarArchive Open(Stream stream, ReaderOptions? options = null)
{
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (stream is not { CanSeek: true })
{
@@ -150,7 +150,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
ReaderOptions? readerOptions = null
)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var files = fileInfos.ToArray();
return new RarArchive(
new SourceStream(
@@ -168,7 +168,7 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
/// <param name="readerOptions"></param>
public static RarArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var strms = streams.ToArray();
return new RarArchive(
new SourceStream(

View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Rar;
using SharpCompress.Common.Rar.Headers;
@@ -66,18 +67,23 @@ public class RarArchiveEntry : RarEntry, IArchiveEntry
}
}
public Stream OpenEntryStream()
public Stream OpenEntryStream() =>
throw new NotSupportedException(
"Synchronous extraction is not supported. Use OpenEntryStreamAsync instead."
);
public async Task<Stream> OpenEntryStreamAsync()
{
if (IsRarV3)
{
return new RarStream(
return await RarStream.Create(
archive.UnpackV1.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)
);
}
return new RarStream(
return await RarStream.Create(
archive.UnpackV2017.Value,
FileHeader,
new MultiVolumeReadOnlyStream(Parts.Cast<RarFilePart>(), archive)

View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.SevenZip;
using SharpCompress.Compressors.LZMA.Utilites;
@@ -21,7 +22,7 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
/// <param name="readerOptions"></param>
public static SevenZipArchive Open(string filePath, ReaderOptions? readerOptions = null)
{
filePath.CheckNotNullOrEmpty("filePath");
filePath.NotNullOrEmpty("filePath");
return Open(new FileInfo(filePath), readerOptions ?? new ReaderOptions());
}
@@ -32,7 +33,7 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
/// <param name="readerOptions"></param>
public static SevenZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull("fileInfo");
fileInfo.NotNull("fileInfo");
return new SevenZipArchive(
new SourceStream(
fileInfo,
@@ -52,7 +53,7 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
ReaderOptions? readerOptions = null
)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var files = fileInfos.ToArray();
return new SevenZipArchive(
new SourceStream(
@@ -73,7 +74,7 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
ReaderOptions? readerOptions = null
)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var strms = streams.ToArray();
return new SevenZipArchive(
new SourceStream(
@@ -91,7 +92,7 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
/// <param name="readerOptions"></param>
public static SevenZipArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull("stream");
stream.NotNull("stream");
if (stream is not { CanSeek: true })
{
@@ -253,8 +254,8 @@ public class SevenZipArchive : AbstractArchive<SevenZipArchiveEntry, SevenZipVol
}
}
protected override EntryStream GetEntryStream() =>
CreateEntryStream(
protected override Task<EntryStream> GetEntryStreamAsync() =>
CreateEntryStreamAsync(
new ReadOnlySubStream(
_currentStream.NotNull("currentStream is not null"),
_currentItem?.Size ?? 0

View File

@@ -1,4 +1,5 @@
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common.SevenZip;
namespace SharpCompress.Archives.SevenZip;
@@ -10,6 +11,9 @@ public class SevenZipArchiveEntry : SevenZipEntry, IArchiveEntry
public Stream OpenEntryStream() => FilePart.GetCompressedStream();
public virtual async Task<Stream> OpenEntryStreamAsync() =>
await Task.FromResult(OpenEntryStream());
public IArchive Archive { get; }
public bool IsComplete => true;

View File

@@ -22,7 +22,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
/// <param name="readerOptions"></param>
public static TarArchive Open(string filePath, ReaderOptions? readerOptions = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
return Open(new FileInfo(filePath), readerOptions ?? new ReaderOptions());
}
@@ -33,7 +33,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
/// <param name="readerOptions"></param>
public static TarArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
fileInfo.NotNull(nameof(fileInfo));
return new TarArchive(
new SourceStream(
fileInfo,
@@ -53,7 +53,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
ReaderOptions? readerOptions = null
)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var files = fileInfos.ToArray();
return new TarArchive(
new SourceStream(
@@ -71,7 +71,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
/// <param name="readerOptions"></param>
public static TarArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var strms = streams.ToArray();
return new TarArchive(
new SourceStream(
@@ -89,7 +89,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
/// <param name="readerOptions"></param>
public static TarArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (stream is not { CanSeek: true })
{
@@ -178,7 +178,7 @@ public class TarArchive : AbstractWritableArchive<TarArchiveEntry, TarVolume>
using (var entryStream = entry.OpenEntryStream())
{
using var memoryStream = new MemoryStream();
entryStream.TransferTo(memoryStream);
entryStream.CopyTo(memoryStream);
memoryStream.Position = 0;
var bytes = memoryStream.ToArray();

View File

@@ -1,5 +1,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Tar;
@@ -12,6 +13,9 @@ public class TarArchiveEntry : TarEntry, IArchiveEntry
public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull();
public virtual async Task<Stream> OpenEntryStreamAsync() =>
await Task.FromResult(OpenEntryStream());
#region IArchiveEntry Members
public IArchive Archive { get; }

View File

@@ -43,7 +43,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
/// <param name="readerOptions"></param>
public static ZipArchive Open(string filePath, ReaderOptions? readerOptions = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
filePath.NotNullOrEmpty(nameof(filePath));
return Open(new FileInfo(filePath), readerOptions ?? new ReaderOptions());
}
@@ -54,7 +54,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
/// <param name="readerOptions"></param>
public static ZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
fileInfo.NotNull(nameof(fileInfo));
return new ZipArchive(
new SourceStream(
fileInfo,
@@ -74,7 +74,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
ReaderOptions? readerOptions = null
)
{
fileInfos.CheckNotNull(nameof(fileInfos));
fileInfos.NotNull(nameof(fileInfos));
var files = fileInfos.ToArray();
return new ZipArchive(
new SourceStream(
@@ -92,7 +92,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
/// <param name="readerOptions"></param>
public static ZipArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
streams.NotNull(nameof(streams));
var strms = streams.ToArray();
return new ZipArchive(
new SourceStream(
@@ -110,7 +110,7 @@ public class ZipArchive : AbstractWritableArchive<ZipArchiveEntry, ZipVolume>
/// <param name="readerOptions"></param>
public static ZipArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
stream.NotNull(nameof(stream));
if (stream is not { CanSeek: true })
{

View File

@@ -1,5 +1,6 @@
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common.Zip;
namespace SharpCompress.Archives.Zip;
@@ -11,6 +12,9 @@ public class ZipArchiveEntry : ZipEntry, IArchiveEntry
public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull();
public virtual async Task<Stream> OpenEntryStreamAsync() =>
await Task.FromResult(OpenEntryStream());
#region IArchiveEntry Members
public IArchive Archive { get; }

View File

@@ -1,7 +1,7 @@
using System;
using System.Runtime.CompilerServices;
[assembly: CLSCompliant(false)]
[assembly: CLSCompliant(true)]
[assembly: InternalsVisibleTo(
"SharpCompress.Test,PublicKey=0024000004800000940000000602000000240000525341310004000001000100158bebf1433f76dffc356733c138babea7a47536c65ed8009b16372c6f4edbb20554db74a62687f56b97c20a6ce8c4b123280279e33c894e7b3aa93ab3c573656fde4db576cfe07dba09619ead26375b25d2c4a8e43f7be257d712b0dd2eb546f67adb09281338618a58ac834fc038dd7e2740a7ab3591826252e4f4516306dc"
)]

View File

@@ -7,53 +7,54 @@ using System.Threading.Tasks;
using SharpCompress.Common.GZip;
using SharpCompress.Common.Tar;
namespace SharpCompress.Common.Arc;
public class ArcEntry : Entry
namespace SharpCompress.Common.Arc
{
private readonly ArcFilePart? _filePart;
internal ArcEntry(ArcFilePart? filePart)
public class ArcEntry : Entry
{
_filePart = filePart;
}
private readonly ArcFilePart? _filePart;
public override long Crc
{
get
internal ArcEntry(ArcFilePart? filePart)
{
if (_filePart == null)
{
return 0;
}
return _filePart.Header.Crc16;
_filePart = filePart;
}
public override long Crc
{
get
{
if (_filePart == null)
{
return 0;
}
return _filePart.Header.Crc16;
}
}
public override string? Key => _filePart?.Header.Name;
public override string? LinkTarget => null;
public override long CompressedSize => _filePart?.Header.CompressedSize ?? 0;
public override CompressionType CompressionType =>
_filePart?.Header.CompressionMethod ?? CompressionType.Unknown;
public override long Size => throw new NotImplementedException();
public override DateTime? LastModifiedTime => null;
public override DateTime? CreatedTime => null;
public override DateTime? LastAccessedTime => null;
public override DateTime? ArchivedTime => null;
public override bool IsEncrypted => false;
public override bool IsDirectory => false;
public override bool IsSplitAfter => false;
internal override IEnumerable<FilePart> Parts => _filePart.Empty();
}
public override string? Key => _filePart?.Header.Name;
public override string? LinkTarget => null;
public override long CompressedSize => _filePart?.Header.CompressedSize ?? 0;
public override CompressionType CompressionType =>
_filePart?.Header.CompressionMethod ?? CompressionType.Unknown;
public override long Size => throw new NotImplementedException();
public override DateTime? LastModifiedTime => null;
public override DateTime? CreatedTime => null;
public override DateTime? LastAccessedTime => null;
public override DateTime? ArchivedTime => null;
public override bool IsEncrypted => false;
public override bool IsDirectory => false;
public override bool IsSplitAfter => false;
internal override IEnumerable<FilePart> Parts => _filePart.Empty();
}

View File

@@ -3,73 +3,74 @@ using System.IO;
using System.Linq;
using System.Text;
namespace SharpCompress.Common.Arc;
public class ArcEntryHeader
namespace SharpCompress.Common.Arc
{
public ArchiveEncoding ArchiveEncoding { get; }
public CompressionType CompressionMethod { get; private set; }
public string? Name { get; private set; }
public long CompressedSize { get; private set; }
public DateTime DateTime { get; private set; }
public int Crc16 { get; private set; }
public long OriginalSize { get; private set; }
public long DataStartPosition { get; private set; }
public ArcEntryHeader(ArchiveEncoding archiveEncoding)
public class ArcEntryHeader
{
this.ArchiveEncoding = archiveEncoding;
}
public ArchiveEncoding ArchiveEncoding { get; }
public CompressionType CompressionMethod { get; private set; }
public string? Name { get; private set; }
public long CompressedSize { get; private set; }
public DateTime DateTime { get; private set; }
public int Crc16 { get; private set; }
public long OriginalSize { get; private set; }
public long DataStartPosition { get; private set; }
public ArcEntryHeader? ReadHeader(Stream stream)
{
byte[] headerBytes = new byte[29];
if (stream.Read(headerBytes, 0, headerBytes.Length) != headerBytes.Length)
public ArcEntryHeader(ArchiveEncoding archiveEncoding)
{
return null;
this.ArchiveEncoding = archiveEncoding;
}
DataStartPosition = stream.Position;
return LoadFrom(headerBytes);
}
public ArcEntryHeader LoadFrom(byte[] headerBytes)
{
CompressionMethod = GetCompressionType(headerBytes[1]);
// Read name
int nameEnd = Array.IndexOf(headerBytes, (byte)0, 1); // Find null terminator
Name = Encoding.UTF8.GetString(headerBytes, 2, nameEnd > 0 ? nameEnd - 2 : 12);
int offset = 15;
CompressedSize = BitConverter.ToUInt32(headerBytes, offset);
offset += 4;
uint rawDateTime = BitConverter.ToUInt32(headerBytes, offset);
DateTime = ConvertToDateTime(rawDateTime);
offset += 4;
Crc16 = BitConverter.ToUInt16(headerBytes, offset);
offset += 2;
OriginalSize = BitConverter.ToUInt32(headerBytes, offset);
return this;
}
private CompressionType GetCompressionType(byte value)
{
return value switch
public ArcEntryHeader? ReadHeader(Stream stream)
{
1 or 2 => CompressionType.None,
3 => CompressionType.RLE90,
4 => CompressionType.Squeezed,
5 or 6 or 7 or 8 => CompressionType.Crunched,
9 => CompressionType.Squashed,
10 => CompressionType.Crushed,
11 => CompressionType.Distilled,
_ => CompressionType.Unknown,
};
}
byte[] headerBytes = new byte[29];
if (stream.Read(headerBytes, 0, headerBytes.Length) != headerBytes.Length)
{
return null;
}
DataStartPosition = stream.Position;
return LoadFrom(headerBytes);
}
public static DateTime ConvertToDateTime(long rawDateTime)
{
// Convert Unix timestamp to DateTime (UTC)
return DateTimeOffset.FromUnixTimeSeconds(rawDateTime).UtcDateTime;
public ArcEntryHeader LoadFrom(byte[] headerBytes)
{
CompressionMethod = GetCompressionType(headerBytes[1]);
// Read name
int nameEnd = Array.IndexOf(headerBytes, (byte)0, 1); // Find null terminator
Name = Encoding.UTF8.GetString(headerBytes, 2, nameEnd > 0 ? nameEnd - 2 : 12);
int offset = 15;
CompressedSize = BitConverter.ToUInt32(headerBytes, offset);
offset += 4;
uint rawDateTime = BitConverter.ToUInt32(headerBytes, offset);
DateTime = ConvertToDateTime(rawDateTime);
offset += 4;
Crc16 = BitConverter.ToUInt16(headerBytes, offset);
offset += 2;
OriginalSize = BitConverter.ToUInt32(headerBytes, offset);
return this;
}
private CompressionType GetCompressionType(byte value)
{
return value switch
{
1 or 2 => CompressionType.None,
3 => CompressionType.RLE90,
4 => CompressionType.Squeezed,
5 or 6 or 7 or 8 => CompressionType.Crunched,
9 => CompressionType.Squashed,
10 => CompressionType.Crushed,
11 => CompressionType.Distilled,
_ => CompressionType.Unknown,
};
}
public static DateTime ConvertToDateTime(long rawDateTime)
{
// Convert Unix timestamp to DateTime (UTC)
return DateTimeOffset.FromUnixTimeSeconds(rawDateTime).UtcDateTime;
}
}
}

View File

@@ -13,55 +13,63 @@ using SharpCompress.Compressors.RLE90;
using SharpCompress.Compressors.Squeezed;
using SharpCompress.IO;
namespace SharpCompress.Common.Arc;
public class ArcFilePart : FilePart
namespace SharpCompress.Common.Arc
{
private readonly Stream? _stream;
internal ArcFilePart(ArcEntryHeader localArcHeader, Stream? seekableStream)
: base(localArcHeader.ArchiveEncoding)
public class ArcFilePart : FilePart
{
_stream = seekableStream;
Header = localArcHeader;
}
private readonly Stream? _stream;
internal ArcEntryHeader Header { get; set; }
internal override string? FilePartName => Header.Name;
internal override Stream GetCompressedStream()
{
if (_stream != null)
internal ArcFilePart(ArcEntryHeader localArcHeader, Stream? seekableStream)
: base(localArcHeader.ArchiveEncoding)
{
Stream compressedStream;
switch (Header.CompressionMethod)
{
case CompressionType.None:
compressedStream = new ReadOnlySubStream(
_stream,
Header.DataStartPosition,
Header.CompressedSize
);
break;
case CompressionType.RLE90:
compressedStream = new RunLength90Stream(_stream, (int)Header.CompressedSize);
break;
case CompressionType.Squeezed:
compressedStream = new SqueezeStream(_stream, (int)Header.CompressedSize);
break;
case CompressionType.Crunched:
compressedStream = new ArcLzwStream(_stream, (int)Header.CompressedSize, true);
break;
default:
throw new NotSupportedException(
"CompressionMethod: " + Header.CompressionMethod
);
}
return compressedStream;
_stream = seekableStream;
Header = localArcHeader;
}
return _stream.NotNull();
}
internal override Stream? GetRawStream() => _stream;
internal ArcEntryHeader Header { get; set; }
internal override string? FilePartName => Header.Name;
internal override Stream GetCompressedStream()
{
if (_stream != null)
{
Stream compressedStream;
switch (Header.CompressionMethod)
{
case CompressionType.None:
compressedStream = new ReadOnlySubStream(
_stream,
Header.DataStartPosition,
Header.CompressedSize
);
break;
case CompressionType.RLE90:
compressedStream = new RunLength90Stream(
_stream,
(int)Header.CompressedSize
);
break;
case CompressionType.Squeezed:
compressedStream = new SqueezeStream(_stream, (int)Header.CompressedSize);
break;
case CompressionType.Crunched:
compressedStream = new ArcLzwStream(
_stream,
(int)Header.CompressedSize,
true
);
break;
default:
throw new NotSupportedException(
"CompressionMethod: " + Header.CompressionMethod
);
}
return compressedStream;
}
return _stream.NotNull();
}
internal override Stream? GetRawStream() => _stream;
}
}

View File

@@ -6,10 +6,11 @@ using System.Text;
using System.Threading.Tasks;
using SharpCompress.Readers;
namespace SharpCompress.Common.Arc;
public class ArcVolume : Volume
namespace SharpCompress.Common.Arc
{
public ArcVolume(Stream stream, ReaderOptions readerOptions, int index = 0)
: base(stream, readerOptions, index) { }
public class ArcVolume : Volume
{
public ArcVolume(Stream stream, ReaderOptions readerOptions, int index = 0)
: base(stream, readerOptions, index) { }
}
}

View File

@@ -1,6 +1,7 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using SharpCompress.IO;
using SharpCompress.Readers;
@@ -51,6 +52,15 @@ public class EntryStream : Stream, IStreamStack
_completed = true;
}
/// <summary>
/// When reading a stream from OpenEntryStream, the stream must be completed so use this to finish reading the entire entry.
/// </summary>
public async Task SkipEntryAsync()
{
await this.SkipAsync();
_completed = true;
}
protected override void Dispose(bool disposing)
{
if (!(_completed || _reader.Cancelled))

View File

@@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Threading.Tasks;
namespace SharpCompress.Common;
@@ -8,11 +9,11 @@ internal static class ExtractionMethods
/// <summary>
/// Extract to specific directory, retaining filename
/// </summary>
public static void WriteEntryToDirectory(
public static async Task WriteEntryToDirectoryAsync(
IEntry entry,
string destinationDirectory,
ExtractionOptions? options,
Action<string, ExtractionOptions?> write
Func<string, ExtractionOptions?, Task> write
)
{
string destinationFileName;
@@ -77,7 +78,7 @@ internal static class ExtractionMethods
"Entry is trying to write a file outside of the destination directory."
);
}
write(destinationFileName, options);
await write(destinationFileName, options);
}
else if (options.ExtractFullPath && !Directory.Exists(destinationFileName))
{
@@ -85,11 +86,11 @@ internal static class ExtractionMethods
}
}
public static void WriteEntryToFile(
public static async Task WriteEntryToFileAsync(
IEntry entry,
string destinationFileName,
ExtractionOptions? options,
Action<string, FileMode> openAndWrite
Func<string, FileMode, Task> openAndWrite
)
{
if (entry.LinkTarget != null)
@@ -112,7 +113,7 @@ internal static class ExtractionMethods
fm = FileMode.CreateNew;
}
openAndWrite(destinationFileName, fm);
await openAndWrite(destinationFileName, fm);
entry.PreserveExtractionOptions(destinationFileName, options);
}
}

View File

@@ -13,8 +13,8 @@ using SharpCompress.Compressors.PPMd;
using SharpCompress.Compressors.Reduce;
using SharpCompress.Compressors.Shrink;
using SharpCompress.Compressors.Xz;
using SharpCompress.Compressors.ZStandard;
using SharpCompress.IO;
using ZstdSharp;
namespace SharpCompress.Common.Zip;

View File

@@ -1,35 +1,36 @@
using System.IO;
namespace SharpCompress.Compressors.Filters;
internal class DeltaFilter : Filter
namespace SharpCompress.Compressors.Filters
{
private const int DISTANCE_MIN = 1;
private const int DISTANCE_MAX = 256;
private const int DISTANCE_MASK = DISTANCE_MAX - 1;
private int _distance;
private byte[] _history;
private int _position;
public DeltaFilter(bool isEncoder, Stream baseStream, byte[] info)
: base(isEncoder, baseStream, 1)
internal class DeltaFilter : Filter
{
_distance = info[0];
_history = new byte[DISTANCE_MAX];
_position = 0;
}
private const int DISTANCE_MIN = 1;
private const int DISTANCE_MAX = 256;
private const int DISTANCE_MASK = DISTANCE_MAX - 1;
protected override int Transform(byte[] buffer, int offset, int count)
{
var end = offset + count;
private int _distance;
private byte[] _history;
private int _position;
for (var i = offset; i < end; i++)
public DeltaFilter(bool isEncoder, Stream baseStream, byte[] info)
: base(isEncoder, baseStream, 1)
{
buffer[i] += _history[(_distance + _position--) & DISTANCE_MASK];
_history[_position & DISTANCE_MASK] = buffer[i];
_distance = info[0];
_history = new byte[DISTANCE_MAX];
_position = 0;
}
return count;
protected override int Transform(byte[] buffer, int offset, int count)
{
var end = offset + count;
for (var i = offset; i < end; i++)
{
buffer[i] += _history[(_distance + _position--) & DISTANCE_MASK];
_history[_position & DISTANCE_MASK] = buffer[i];
}
return count;
}
}
}

View File

@@ -1,11 +1,12 @@
#nullable disable
using System;
using System.Buffers;
using System.IO;
namespace SharpCompress.Compressors.LZMA.LZ;
internal class OutWindow
internal class OutWindow : IDisposable
{
private byte[] _buffer;
private int _windowSize;
@@ -15,19 +16,22 @@ internal class OutWindow
private int _pendingDist;
private Stream _stream;
public long _total;
public long _limit;
private long _total;
private long _limit;
public long Total => _total;
public void Create(int windowSize)
{
if (_windowSize != windowSize)
{
_buffer = new byte[windowSize];
}
else
{
_buffer[windowSize - 1] = 0;
if (_buffer is not null)
{
ArrayPool<byte>.Shared.Return(_buffer);
}
_buffer = ArrayPool<byte>.Shared.Rent(windowSize);
}
_buffer[windowSize - 1] = 0;
_windowSize = windowSize;
_pos = 0;
_streamPos = 0;
@@ -36,7 +40,22 @@ internal class OutWindow
_limit = 0;
}
public void Reset() => Create(_windowSize);
public void Dispose()
{
ReleaseStream();
if (_buffer is null)
{
return;
}
ArrayPool<byte>.Shared.Return(_buffer);
_buffer = null;
}
public void Reset()
{
ReleaseStream();
Create(_windowSize);
}
public void Init(Stream stream)
{
@@ -66,7 +85,7 @@ internal class OutWindow
_stream = null;
}
public void Flush()
private void Flush()
{
if (_stream is null)
{

View File

@@ -294,7 +294,7 @@ public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream
}
else
{
_outWindow.SetLimit(long.MaxValue - _outWindow._total);
_outWindow.SetLimit(long.MaxValue - _outWindow.Total);
}
var rangeDecoder = new RangeCoder.Decoder();
@@ -305,6 +305,7 @@ public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream
_outWindow.ReleaseStream();
rangeDecoder.ReleaseStream();
_outWindow.Dispose();
_outWindow = null;
}
@@ -316,7 +317,7 @@ public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream
while (outWindow.HasSpace)
{
var posState = (uint)outWindow._total & _posStateMask;
var posState = (uint)outWindow.Total & _posStateMask;
if (
_isMatchDecoders[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState]
.Decode(rangeDecoder) == 0
@@ -328,18 +329,14 @@ public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream
{
b = _literalDecoder.DecodeWithMatchByte(
rangeDecoder,
(uint)outWindow._total,
(uint)outWindow.Total,
prevByte,
outWindow.GetByte((int)_rep0)
);
}
else
{
b = _literalDecoder.DecodeNormal(
rangeDecoder,
(uint)outWindow._total,
prevByte
);
b = _literalDecoder.DecodeNormal(rangeDecoder, (uint)outWindow.Total, prevByte);
}
outWindow.PutByte(b);
_state.UpdateChar();
@@ -424,7 +421,7 @@ public class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream
_rep0 = posSlot;
}
}
if (_rep0 >= outWindow._total || _rep0 >= dictionarySizeCheck)
if (_rep0 >= outWindow.Total || _rep0 >= dictionarySizeCheck)
{
if (_rep0 == 0xFFFFFFFF)
{

View File

@@ -178,6 +178,7 @@ public class LzmaStream : Stream, IStreamStack
_position = _encoder.Code(null, true);
}
_inputStream?.Dispose();
_outWindow.Dispose();
}
base.Dispose(disposing);
}

View File

@@ -7,7 +7,7 @@ using SharpCompress.Compressors.Deflate;
using SharpCompress.Compressors.Filters;
using SharpCompress.Compressors.LZMA.Utilites;
using SharpCompress.Compressors.PPMd;
using SharpCompress.Compressors.ZStandard;
using ZstdSharp;
namespace SharpCompress.Compressors.LZMA;

View File

@@ -1,64 +1,65 @@
namespace SharpCompress.Compressors.Lzw;
/// <summary>
/// This class contains constants used for LZW
/// </summary>
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Naming",
"CA1707:Identifiers should not contain underscores",
Justification = "kept for backwards compatibility"
)]
public sealed class LzwConstants
namespace SharpCompress.Compressors.Lzw
{
/// <summary>
/// Magic number found at start of LZW header: 0x1f 0x9d
/// This class contains constants used for LZW
/// </summary>
public const int MAGIC = 0x1f9d;
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Naming",
"CA1707:Identifiers should not contain underscores",
Justification = "kept for backwards compatibility"
)]
public sealed class LzwConstants
{
/// <summary>
/// Magic number found at start of LZW header: 0x1f 0x9d
/// </summary>
public const int MAGIC = 0x1f9d;
/// <summary>
/// Maximum number of bits per code
/// </summary>
public const int MAX_BITS = 16;
/// <summary>
/// Maximum number of bits per code
/// </summary>
public const int MAX_BITS = 16;
/* 3rd header byte:
* bit 0..4 Number of compression bits
* bit 5 Extended header
* bit 6 Free
* bit 7 Block mode
*/
/* 3rd header byte:
* bit 0..4 Number of compression bits
* bit 5 Extended header
* bit 6 Free
* bit 7 Block mode
*/
/// <summary>
/// Mask for 'number of compression bits'
/// </summary>
public const int BIT_MASK = 0x1f;
/// <summary>
/// Mask for 'number of compression bits'
/// </summary>
public const int BIT_MASK = 0x1f;
/// <summary>
/// Indicates the presence of a fourth header byte
/// </summary>
public const int EXTENDED_MASK = 0x20;
/// <summary>
/// Indicates the presence of a fourth header byte
/// </summary>
public const int EXTENDED_MASK = 0x20;
//public const int FREE_MASK = 0x40;
//public const int FREE_MASK = 0x40;
/// <summary>
/// Reserved bits
/// </summary>
public const int RESERVED_MASK = 0x60;
/// <summary>
/// Reserved bits
/// </summary>
public const int RESERVED_MASK = 0x60;
/// <summary>
/// Block compression: if table is full and compression rate is dropping,
/// clear the dictionary.
/// </summary>
public const int BLOCK_MODE_MASK = 0x80;
/// <summary>
/// Block compression: if table is full and compression rate is dropping,
/// clear the dictionary.
/// </summary>
public const int BLOCK_MODE_MASK = 0x80;
/// <summary>
/// LZW file header size (in bytes)
/// </summary>
public const int HDR_SIZE = 3;
/// <summary>
/// LZW file header size (in bytes)
/// </summary>
public const int HDR_SIZE = 3;
/// <summary>
/// Initial number of bits per code
/// </summary>
public const int INIT_BITS = 9;
/// <summary>
/// Initial number of bits per code
/// </summary>
public const int INIT_BITS = 9;
private LzwConstants() { }
private LzwConstants() { }
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,51 +1,52 @@
using System.Collections.Generic;
using System.Linq;
namespace SharpCompress.Compressors.RLE90;
public static class RLE
namespace SharpCompress.Compressors.RLE90
{
private const byte DLE = 0x90;
/// <summary>
/// Unpacks an RLE compressed buffer.
/// Format: <char> DLE <count>, where count == 0 -> DLE
/// </summary>
/// <param name="compressedBuffer">The compressed buffer to unpack.</param>
/// <returns>A list of unpacked bytes.</returns>
public static List<byte> UnpackRLE(byte[] compressedBuffer)
public static class RLE
{
var result = new List<byte>(compressedBuffer.Length * 2); // Optimized initial capacity
var countMode = false;
byte last = 0;
private const byte DLE = 0x90;
foreach (var c in compressedBuffer)
/// <summary>
/// Unpacks an RLE compressed buffer.
/// Format: <char> DLE <count>, where count == 0 -> DLE
/// </summary>
/// <param name="compressedBuffer">The compressed buffer to unpack.</param>
/// <returns>A list of unpacked bytes.</returns>
public static List<byte> UnpackRLE(byte[] compressedBuffer)
{
if (!countMode)
var result = new List<byte>(compressedBuffer.Length * 2); // Optimized initial capacity
var countMode = false;
byte last = 0;
foreach (var c in compressedBuffer)
{
if (c == DLE)
if (!countMode)
{
countMode = true;
if (c == DLE)
{
countMode = true;
}
else
{
result.Add(c);
last = c;
}
}
else
{
result.Add(c);
last = c;
}
}
else
{
countMode = false;
if (c == 0)
{
result.Add(DLE);
}
else
{
result.AddRange(Enumerable.Repeat(last, c - 1));
countMode = false;
if (c == 0)
{
result.Add(DLE);
}
else
{
result.AddRange(Enumerable.Repeat(last, c - 1));
}
}
}
return result;
}
return result;
}
}

View File

@@ -6,90 +6,91 @@ using System.Text;
using System.Threading.Tasks;
using SharpCompress.IO;
namespace SharpCompress.Compressors.RLE90;
public class RunLength90Stream : Stream, IStreamStack
namespace SharpCompress.Compressors.RLE90
{
#if DEBUG_STREAMS
long IStreamStack.InstanceId { get; set; }
#endif
int IStreamStack.DefaultBufferSize { get; set; }
Stream IStreamStack.BaseStream() => _stream;
int IStreamStack.BufferSize
{
get => 0;
set { }
}
int IStreamStack.BufferPosition
{
get => 0;
set { }
}
void IStreamStack.SetPosition(long position) { }
private readonly Stream _stream;
private const byte DLE = 0x90;
private int _compressedSize;
private bool _processed = false;
public RunLength90Stream(Stream stream, int compressedSize)
{
_stream = stream;
_compressedSize = compressedSize;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RunLength90Stream));
#endif
}
protected override void Dispose(bool disposing)
public class RunLength90Stream : Stream, IStreamStack
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(RunLength90Stream));
long IStreamStack.InstanceId { get; set; }
#endif
base.Dispose(disposing);
}
int IStreamStack.DefaultBufferSize { get; set; }
public override bool CanRead => true;
Stream IStreamStack.BaseStream() => _stream;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotImplementedException();
public override long Position
{
get => _stream.Position;
set => throw new NotImplementedException();
}
public override void Flush() => throw new NotImplementedException();
public override int Read(byte[] buffer, int offset, int count)
{
if (_processed)
int IStreamStack.BufferSize
{
return 0;
get => 0;
set { }
}
int IStreamStack.BufferPosition
{
get => 0;
set { }
}
_processed = true;
using var binaryReader = new BinaryReader(_stream);
byte[] compressedBuffer = binaryReader.ReadBytes(_compressedSize);
void IStreamStack.SetPosition(long position) { }
var unpacked = RLE.UnpackRLE(compressedBuffer);
unpacked.CopyTo(buffer);
private readonly Stream _stream;
private const byte DLE = 0x90;
private int _compressedSize;
private bool _processed = false;
return unpacked.Count;
public RunLength90Stream(Stream stream, int compressedSize)
{
_stream = stream;
_compressedSize = compressedSize;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RunLength90Stream));
#endif
}
protected override void Dispose(bool disposing)
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(RunLength90Stream));
#endif
base.Dispose(disposing);
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotImplementedException();
public override long Position
{
get => _stream.Position;
set => throw new NotImplementedException();
}
public override void Flush() => throw new NotImplementedException();
public override int Read(byte[] buffer, int offset, int count)
{
if (_processed)
{
return 0;
}
_processed = true;
using var binaryReader = new BinaryReader(_stream);
byte[] compressedBuffer = binaryReader.ReadBytes(_compressedSize);
var unpacked = RLE.UnpackRLE(compressedBuffer);
unpacked.CopyTo(buffer);
return unpacked.Count;
}
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();
public override void SetLength(long value) => throw new NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotImplementedException();
}
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();
public override void SetLength(long value) => throw new NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotImplementedException();
}

View File

@@ -1,12 +1,18 @@
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common.Rar.Headers;
namespace SharpCompress.Compressors.Rar;
internal interface IRarUnpack
{
#if NETSTANDARD2_0 || NETFRAMEWORK
void DoUnpack(FileHeader fileHeader, Stream readStream, Stream writeStream);
void DoUnpack();
#else
ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream);
ValueTask DoUnpackAsync();
#endif
// eg u/i pause/resume button
bool Suspended { get; set; }

View File

@@ -1,6 +1,7 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.IO;
@@ -9,6 +10,17 @@ namespace SharpCompress.Compressors.Rar;
internal class RarBLAKE2spStream : RarStream, IStreamStack
{
public static async ValueTask<RarBLAKE2spStream> Create(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream
)
{
var rs = new RarBLAKE2spStream(unpack, fileHeader, readStream);
await RarStream.Initialize(rs, unpack, fileHeader, readStream);
return rs;
}
#if DEBUG_STREAMS
long IStreamStack.InstanceId { get; set; }
#endif
@@ -103,7 +115,7 @@ internal class RarBLAKE2spStream : RarStream, IStreamStack
byte[] _hash = { };
public RarBLAKE2spStream(
protected RarBLAKE2spStream(
IRarUnpack unpack,
FileHeader fileHeader,
MultiVolumeReadOnlyStream readStream

View File

@@ -1,8 +1,8 @@
#nullable disable
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.IO;
@@ -36,18 +36,47 @@ internal class RarStream : Stream, IStreamStack
private bool fetch;
private byte[] tmpBuffer = ArrayPool<byte>.Shared.Rent(65536);
private byte[]? tmpBuffer = ArrayPool<byte>.Shared.Rent(65536);
private int tmpOffset;
private int tmpCount;
private byte[] outBuffer;
private byte[]? outBuffer;
private int outOffset;
private int outCount;
private int outTotal;
private bool isDisposed;
private long _position;
public RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream)
public static async ValueTask<RarStream> Create(
IRarUnpack unpack,
FileHeader fileHeader,
Stream readStream
)
{
var rs = new RarStream(unpack, fileHeader, readStream);
await Initialize(rs, unpack, fileHeader, readStream);
return rs;
}
internal static async ValueTask Initialize(
RarStream rs,
IRarUnpack unpack,
FileHeader fileHeader,
Stream readStream
)
{
rs.fetch = true;
#if !NETSTANDARD2_0 && !NETFRAMEWORK
await unpack.DoUnpackAsync(fileHeader, readStream, rs);
#else
unpack.DoUnpack(fileHeader, readStream, rs);
await Task.CompletedTask;
#endif
rs.fetch = false;
rs._position = 0;
}
protected RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream)
{
this.unpack = unpack;
this.fileHeader = fileHeader;
@@ -56,11 +85,6 @@ internal class RarStream : Stream, IStreamStack
#if DEBUG_STREAMS
this.DebugConstruct(typeof(RarStream));
#endif
fetch = true;
unpack.DoUnpack(fileHeader, readStream, this);
fetch = false;
_position = 0;
}
protected override void Dispose(bool disposing)
@@ -72,8 +96,11 @@ internal class RarStream : Stream, IStreamStack
#if DEBUG_STREAMS
this.DebugDispose(typeof(RarStream));
#endif
ArrayPool<byte>.Shared.Return(this.tmpBuffer);
this.tmpBuffer = null;
if (tmpBuffer != null)
{
ArrayPool<byte>.Shared.Return(this.tmpBuffer);
this.tmpBuffer = null;
}
}
isDisposed = true;
base.Dispose(disposing);
@@ -81,6 +108,26 @@ internal class RarStream : Stream, IStreamStack
}
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override async ValueTask DisposeAsync()
{
if (!isDisposed)
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(RarStream));
#endif
if (tmpBuffer != null)
{
ArrayPool<byte>.Shared.Return(this.tmpBuffer);
this.tmpBuffer = null;
}
isDisposed = true;
await readStream.DisposeAsync().ConfigureAwait(false);
}
await base.DisposeAsync().ConfigureAwait(false);
}
#endif
public override bool CanRead => true;
public override bool CanSeek => false;
@@ -89,6 +136,8 @@ internal class RarStream : Stream, IStreamStack
public override void Flush() { }
public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public override long Length => fileHeader.UncompressedSize;
//commented out code always returned the length of the file
@@ -98,8 +147,96 @@ internal class RarStream : Stream, IStreamStack
set => throw new NotSupportedException();
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var bytesRead = Read(buffer, offset, count);
return Task.FromResult(bytesRead);
}
catch (Exception ex)
{
return Task.FromException<int>(ex);
}
}
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
outTotal = 0;
var count = buffer.Length;
var offset = 0;
if (tmpCount > 0)
{
var toCopy = tmpCount < count ? tmpCount : count;
tmpBuffer.AsSpan(tmpOffset, toCopy).CopyTo(buffer.Span.Slice(offset, toCopy));
tmpOffset += toCopy;
tmpCount -= toCopy;
offset += toCopy;
count -= toCopy;
outTotal += toCopy;
}
if (count > 0 && unpack.DestSize > 0)
{
// Create a temporary array for the unpack operation
var tempArray = ArrayPool<byte>.Shared.Rent(count);
try
{
outBuffer = tempArray;
outOffset = 0;
outCount = count;
fetch = true;
await unpack.DoUnpackAsync();
fetch = false;
// Copy the unpacked data to the memory buffer
var unpacked = outTotal - (tmpCount > 0 ? offset : 0);
if (unpacked > 0)
{
tempArray.AsSpan(0, unpacked).CopyTo(buffer.Span.Slice(offset, unpacked));
}
}
finally
{
ArrayPool<byte>.Shared.Return(tempArray);
outBuffer = null;
}
}
_position += outTotal;
if (count > 0 && outTotal == 0 && _position != Length)
{
// sanity check, eg if we try to decompress a redir entry
throw new InvalidOperationException(
$"unpacked file size does not match header: expected {Length} found {_position}"
);
}
return outTotal;
}
public override int Read(byte[] buffer, int offset, int count) =>
throw new NotSupportedException("Use ReadAsync or ReadAsync(Memory<byte>) instead.");
#else
public override int Read(byte[] buffer, int offset, int count)
{
if (tmpBuffer == null)
{
throw new ObjectDisposedException(nameof(RarStream));
}
outTotal = 0;
if (tmpCount > 0)
{
@@ -130,6 +267,7 @@ internal class RarStream : Stream, IStreamStack
}
return outTotal;
}
#endif
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
@@ -137,6 +275,14 @@ internal class RarStream : Stream, IStreamStack
public override void Write(byte[] buffer, int offset, int count)
{
if (tmpBuffer == null)
{
throw new ObjectDisposedException(nameof(RarStream));
}
if (outBuffer == null)
{
throw new ObjectDisposedException(nameof(RarStream));
}
if (!fetch)
{
throw new NotSupportedException();
@@ -165,8 +311,81 @@ internal class RarStream : Stream, IStreamStack
}
}
public override Task WriteAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
Write(buffer, offset, count);
return Task.CompletedTask;
}
catch (Exception ex)
{
return Task.FromException(ex);
}
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
if (!fetch)
{
throw new NotSupportedException();
}
var count = buffer.Length;
var offset = 0;
if (outCount > 0)
{
var toCopy = outCount < count ? outCount : count;
buffer.Span.Slice(offset, toCopy).CopyTo(outBuffer.AsSpan(outOffset, toCopy));
outOffset += toCopy;
outCount -= toCopy;
offset += toCopy;
count -= toCopy;
outTotal += toCopy;
}
if (count > 0)
{
EnsureBufferCapacity(count);
buffer.Span.Slice(offset, count).CopyTo(tmpBuffer.AsSpan(tmpCount, count));
tmpCount += count;
tmpOffset = 0;
unpack.Suspended = true;
}
else
{
unpack.Suspended = false;
}
return ValueTask.CompletedTask;
}
catch (Exception ex)
{
return new ValueTask(Task.FromException(ex));
}
}
#endif
private void EnsureBufferCapacity(int count)
{
if (tmpBuffer == null)
{
throw new ObjectDisposedException(nameof(RarStream));
}
if (this.tmpBuffer.Length < this.tmpCount + count)
{
var newLength =

View File

@@ -4,6 +4,7 @@ using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.Compressors.PPMd.H;
@@ -13,7 +14,7 @@ using SharpCompress.Compressors.Rar.VM;
namespace SharpCompress.Compressors.Rar.UnpackV1;
internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
internal sealed partial class Unpack : BitInput, IRarUnpack
{
private readonly BitInput Inp;
private bool disposed;
@@ -22,15 +23,17 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
// to ease in porting Unpack50.cs
Inp = this;
public void Dispose()
public override void Dispose()
{
if (!disposed)
{
base.Dispose();
if (!externalWindow)
{
ArrayPool<byte>.Shared.Return(window);
window = null;
}
rarVM.Dispose();
disposed = true;
}
}
@@ -153,6 +156,20 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
DoUnpack();
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public ValueTask DoUnpackAsync()
{
DoUnpack();
return ValueTask.CompletedTask;
}
public ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream)
{
DoUnpack(fileHeader, readStream, writeStream);
return ValueTask.CompletedTask;
}
#endif
public void DoUnpack()
{
if (fileHeader.CompressionMethod == 0)
@@ -574,104 +591,111 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
var FilteredDataOffset = Prg.FilteredDataOffset;
var FilteredDataSize = Prg.FilteredDataSize;
var FilteredData = new byte[FilteredDataSize];
for (var i = 0; i < FilteredDataSize; i++)
var FilteredData = ArrayPool<byte>.Shared.Rent(FilteredDataSize);
try
{
FilteredData[i] = rarVM.Mem[FilteredDataOffset + i];
Array.Copy(
rarVM.Mem,
FilteredDataOffset,
FilteredData,
0,
FilteredDataSize
);
// Prg.GlobalData.get(FilteredDataOffset
// +
// i);
}
prgStack[I] = null;
while (I + 1 < prgStack.Count)
{
var NextFilter = prgStack[I + 1];
if (
NextFilter is null
|| NextFilter.BlockStart != BlockStart
|| NextFilter.BlockLength != FilteredDataSize
|| NextFilter.NextWindow
)
{
break;
}
// apply several filters to same data block
rarVM.setMemory(0, FilteredData, 0, FilteredDataSize);
// .SetMemory(0,FilteredData,FilteredDataSize);
var pPrg = filters[NextFilter.ParentFilter].Program;
var NextPrg = NextFilter.Program;
if (pPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE)
{
// copy global data from previous script execution
// if any
// NextPrg->GlobalData.Alloc(ParentPrg->GlobalData.Size());
NextPrg.GlobalData.SetSize(pPrg.GlobalData.Count);
// memcpy(&NextPrg->GlobalData[VM_FIXEDGLOBALSIZE],&ParentPrg->GlobalData[VM_FIXEDGLOBALSIZE],ParentPrg->GlobalData.Size()-VM_FIXEDGLOBALSIZE);
for (
var i = 0;
i < pPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE;
i++
)
{
NextPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = pPrg.GlobalData[
RarVM.VM_FIXEDGLOBALSIZE + i
];
}
}
ExecuteCode(NextPrg);
if (NextPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE)
{
// save global data for next script execution
if (pPrg.GlobalData.Count < NextPrg.GlobalData.Count)
{
pPrg.GlobalData.SetSize(NextPrg.GlobalData.Count);
}
// memcpy(&ParentPrg->GlobalData[VM_FIXEDGLOBALSIZE],&NextPrg->GlobalData[VM_FIXEDGLOBALSIZE],NextPrg->GlobalData.Size()-VM_FIXEDGLOBALSIZE);
for (
var i = 0;
i < NextPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE;
i++
)
{
pPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] = NextPrg.GlobalData[
RarVM.VM_FIXEDGLOBALSIZE + i
];
}
}
else
{
pPrg.GlobalData.Clear();
}
FilteredDataOffset = NextPrg.FilteredDataOffset;
FilteredDataSize = NextPrg.FilteredDataSize;
FilteredData = new byte[FilteredDataSize];
for (var i = 0; i < FilteredDataSize; i++)
{
FilteredData[i] = NextPrg.GlobalData[FilteredDataOffset + i];
}
I++;
prgStack[I] = null;
while (I + 1 < prgStack.Count)
{
var NextFilter = prgStack[I + 1];
if (
NextFilter is null
|| NextFilter.BlockStart != BlockStart
|| NextFilter.BlockLength != FilteredDataSize
|| NextFilter.NextWindow
)
{
break;
}
// apply several filters to same data block
rarVM.setMemory(0, FilteredData, 0, FilteredDataSize);
// .SetMemory(0,FilteredData,FilteredDataSize);
var pPrg = filters[NextFilter.ParentFilter].Program;
var NextPrg = NextFilter.Program;
if (pPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE)
{
// copy global data from previous script execution
// if any
// NextPrg->GlobalData.Alloc(ParentPrg->GlobalData.Size());
NextPrg.GlobalData.SetSize(pPrg.GlobalData.Count);
// memcpy(&NextPrg->GlobalData[VM_FIXEDGLOBALSIZE],&ParentPrg->GlobalData[VM_FIXEDGLOBALSIZE],ParentPrg->GlobalData.Size()-VM_FIXEDGLOBALSIZE);
for (
var i = 0;
i < pPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE;
i++
)
{
NextPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] =
pPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i];
}
}
ExecuteCode(NextPrg);
if (NextPrg.GlobalData.Count > RarVM.VM_FIXEDGLOBALSIZE)
{
// save global data for next script execution
if (pPrg.GlobalData.Count < NextPrg.GlobalData.Count)
{
pPrg.GlobalData.SetSize(NextPrg.GlobalData.Count);
}
// memcpy(&ParentPrg->GlobalData[VM_FIXEDGLOBALSIZE],&NextPrg->GlobalData[VM_FIXEDGLOBALSIZE],NextPrg->GlobalData.Size()-VM_FIXEDGLOBALSIZE);
for (
var i = 0;
i < NextPrg.GlobalData.Count - RarVM.VM_FIXEDGLOBALSIZE;
i++
)
{
pPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i] =
NextPrg.GlobalData[RarVM.VM_FIXEDGLOBALSIZE + i];
}
}
else
{
pPrg.GlobalData.Clear();
}
FilteredDataOffset = NextPrg.FilteredDataOffset;
FilteredDataSize = NextPrg.FilteredDataSize;
if (FilteredData.Length < FilteredDataSize)
{
ArrayPool<byte>.Shared.Return(FilteredData);
FilteredData = ArrayPool<byte>.Shared.Rent(FilteredDataSize);
}
for (var i = 0; i < FilteredDataSize; i++)
{
FilteredData[i] = NextPrg.GlobalData[FilteredDataOffset + i];
}
I++;
prgStack[I] = null;
}
writeStream.Write(FilteredData, 0, FilteredDataSize);
writtenFileSize += FilteredDataSize;
destUnpSize -= FilteredDataSize;
WrittenBorder = BlockEnd;
WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK;
}
finally
{
ArrayPool<byte>.Shared.Return(FilteredData);
}
writeStream.Write(FilteredData, 0, FilteredDataSize);
unpSomeRead = true;
writtenFileSize += FilteredDataSize;
destUnpSize -= FilteredDataSize;
WrittenBorder = BlockEnd;
WriteSize = (unpPtr - WrittenBorder) & PackDef.MAXWINMASK;
}
else
{
@@ -695,15 +719,10 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
private void UnpWriteArea(int startPtr, int endPtr)
{
if (endPtr != startPtr)
{
unpSomeRead = true;
}
if (endPtr < startPtr)
{
UnpWriteData(window, startPtr, -startPtr & PackDef.MAXWINMASK);
UnpWriteData(window, 0, endPtr);
unpAllBuf = true;
}
else
{
@@ -757,19 +776,27 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
// System.out.println("copyString(" + length + ", " + distance + ")");
var destPtr = unpPtr - distance;
var safeZone = PackDef.MAXWINSIZE - 260;
// System.out.println(unpPtr+":"+distance);
if (destPtr >= 0 && destPtr < PackDef.MAXWINSIZE - 260 && unpPtr < PackDef.MAXWINSIZE - 260)
// Fast path: use Array.Copy for bulk operations when in safe zone
if (destPtr >= 0 && destPtr < safeZone && unpPtr < safeZone && distance >= length)
{
window[unpPtr++] = window[destPtr++];
while (--length > 0)
// Non-overlapping copy: can use Array.Copy directly
Array.Copy(window, destPtr, window, unpPtr, length);
unpPtr += length;
}
else if (destPtr >= 0 && destPtr < safeZone && unpPtr < safeZone)
{
// Overlapping copy in safe zone: use byte-by-byte to handle self-referential copies
for (int i = 0; i < length; i++)
{
window[unpPtr++] = window[destPtr++];
window[unpPtr + i] = window[destPtr + i];
}
unpPtr += length;
}
else
{
// Slow path with wraparound mask
while (length-- != 0)
{
window[unpPtr] = window[destPtr++ & PackDef.MAXWINMASK];
@@ -1028,7 +1055,7 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
vmCode.Add((byte)(GetBits() >> 8));
AddBits(8);
}
return (AddVMCode(FirstByte, vmCode, Length));
return AddVMCode(FirstByte, vmCode);
}
private bool ReadVMCodePPM()
@@ -1073,12 +1100,12 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
}
vmCode.Add((byte)Ch); // VMCode[I]=Ch;
}
return (AddVMCode(FirstByte, vmCode, Length));
return AddVMCode(FirstByte, vmCode);
}
private bool AddVMCode(int firstByte, List<byte> vmCode, int length)
private bool AddVMCode(int firstByte, List<byte> vmCode)
{
var Inp = new BitInput();
using var Inp = new BitInput();
Inp.InitBitInput();
// memcpy(Inp.InBuf,Code,Min(BitInput::MAX_SIZE,CodeSize));
@@ -1086,7 +1113,6 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
{
Inp.InBuf[i] = vmCode[i];
}
rarVM.init();
int FiltPos;
if ((firstByte & 0x80) != 0)
@@ -1199,19 +1225,28 @@ internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
{
return (false);
}
Span<byte> VMCode = stackalloc byte[VMCodeSize];
for (var I = 0; I < VMCodeSize; I++)
{
if (Inp.Overflow(3))
{
return (false);
}
VMCode[I] = (byte)(Inp.GetBits() >> 8);
Inp.AddBits(8);
}
// VM.Prepare(&VMCode[0],VMCodeSize,&Filter->Prg);
rarVM.prepare(VMCode, VMCodeSize, Filter.Program);
var VMCode = ArrayPool<byte>.Shared.Rent(VMCodeSize);
try
{
for (var I = 0; I < VMCodeSize; I++)
{
if (Inp.Overflow(3))
{
return (false);
}
VMCode[I] = (byte)(Inp.GetBits() >> 8);
Inp.AddBits(8);
}
// VM.Prepare(&VMCode[0],VMCodeSize,&Filter->Prg);
rarVM.prepare(VMCode.AsSpan(0, VMCodeSize), Filter.Program);
}
finally
{
ArrayPool<byte>.Shared.Return(VMCode);
}
}
StackFilter.Program.AltCommands = Filter.Program.Commands; // StackFilter->Prg.AltCmd=&Filter->Prg.Cmd[0];
StackFilter.Program.CommandCount = Filter.Program.CommandCount;

View File

@@ -19,14 +19,9 @@ internal partial class Unpack
private bool suspended;
internal bool unpAllBuf;
//private ComprDataIO unpIO;
private Stream readStream;
private Stream writeStream;
internal bool unpSomeRead;
private int readTop;
private long destUnpSize;
@@ -808,15 +803,10 @@ internal partial class Unpack
private void oldUnpWriteBuf()
{
if (unpPtr != wrPtr)
{
unpSomeRead = true;
}
if (unpPtr < wrPtr)
{
writeStream.Write(window, wrPtr, -wrPtr & PackDef.MAXWINMASK);
writeStream.Write(window, 0, unpPtr);
unpAllBuf = true;
}
else
{

View File

@@ -1,4 +1,5 @@
using System;
using System.Runtime.CompilerServices;
using SharpCompress.Compressors.Rar.VM;
namespace SharpCompress.Compressors.Rar.UnpackV1;
@@ -9,167 +10,15 @@ internal static class UnpackUtility
internal static uint DecodeNumber(this BitInput input, Decode.Decode dec) =>
(uint)input.decodeNumber(dec);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int decodeNumber(this BitInput input, Decode.Decode dec)
{
int bits;
long bitField = input.GetBits() & 0xfffe;
// if (bitField < dec.getDecodeLen()[8]) {
// if (bitField < dec.getDecodeLen()[4]) {
// if (bitField < dec.getDecodeLen()[2]) {
// if (bitField < dec.getDecodeLen()[1]) {
// bits = 1;
// } else {
// bits = 2;
// }
// } else {
// if (bitField < dec.getDecodeLen()[3]) {
// bits = 3;
// } else {
// bits = 4;
// }
// }
// } else {
// if (bitField < dec.getDecodeLen()[6]) {
// if (bitField < dec.getDecodeLen()[5])
// bits = 5;
// else
// bits = 6;
// } else {
// if (bitField < dec.getDecodeLen()[7]) {
// bits = 7;
// } else {
// bits = 8;
// }
// }
// }
// } else {
// if (bitField < dec.getDecodeLen()[12]) {
// if (bitField < dec.getDecodeLen()[10])
// if (bitField < dec.getDecodeLen()[9])
// bits = 9;
// else
// bits = 10;
// else if (bitField < dec.getDecodeLen()[11])
// bits = 11;
// else
// bits = 12;
// } else {
// if (bitField < dec.getDecodeLen()[14]) {
// if (bitField < dec.getDecodeLen()[13]) {
// bits = 13;
// } else {
// bits = 14;
// }
// } else {
// bits = 15;
// }
// }
// }
// addbits(bits);
// int N = dec.getDecodePos()[bits]
// + (((int) bitField - dec.getDecodeLen()[bits - 1]) >>> (16 - bits));
// if (N >= dec.getMaxNum()) {
// N = 0;
// }
// return (dec.getDecodeNum()[N]);
var decodeLen = dec.DecodeLen;
if (bitField < decodeLen[8])
{
if (bitField < decodeLen[4])
{
if (bitField < decodeLen[2])
{
if (bitField < decodeLen[1])
{
bits = 1;
}
else
{
bits = 2;
}
}
else
{
if (bitField < decodeLen[3])
{
bits = 3;
}
else
{
bits = 4;
}
}
}
else
{
if (bitField < decodeLen[6])
{
if (bitField < decodeLen[5])
{
bits = 5;
}
else
{
bits = 6;
}
}
else
{
if (bitField < decodeLen[7])
{
bits = 7;
}
else
{
bits = 8;
}
}
}
}
else
{
if (bitField < decodeLen[12])
{
if (bitField < decodeLen[10])
{
if (bitField < decodeLen[9])
{
bits = 9;
}
else
{
bits = 10;
}
}
else if (bitField < decodeLen[11])
{
bits = 11;
}
else
{
bits = 12;
}
}
else
{
if (bitField < decodeLen[14])
{
if (bitField < decodeLen[13])
{
bits = 13;
}
else
{
bits = 14;
}
}
else
{
bits = 15;
}
}
}
// Binary search to find the bit length - faster than nested ifs
int bits = FindDecodeBits(bitField, decodeLen);
input.AddBits(bits);
var N =
dec.DecodePos[bits]
@@ -181,6 +30,52 @@ internal static class UnpackUtility
return (dec.DecodeNum[N]);
}
/// <summary>
/// Fast binary search to find which bit length matches the bitField.
/// Optimized with cached array access to minimize memory lookups.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int FindDecodeBits(long bitField, int[] decodeLen)
{
// Cache critical values to reduce array access overhead
long len4 = decodeLen[4];
long len8 = decodeLen[8];
long len12 = decodeLen[12];
if (bitField < len8)
{
if (bitField < len4)
{
long len2 = decodeLen[2];
if (bitField < len2)
{
return bitField < decodeLen[1] ? 1 : 2;
}
return bitField < decodeLen[3] ? 3 : 4;
}
long len6 = decodeLen[6];
if (bitField < len6)
{
return bitField < decodeLen[5] ? 5 : 6;
}
return bitField < decodeLen[7] ? 7 : 8;
}
if (bitField < len12)
{
long len10 = decodeLen[10];
if (bitField < len10)
{
return bitField < decodeLen[9] ? 9 : 10;
}
return bitField < decodeLen[11] ? 11 : 12;
}
long len14 = decodeLen[14];
return bitField < len14 ? (bitField < decodeLen[13] ? 13 : 14) : 15;
}
internal static void makeDecodeTables(
Span<byte> lenTab,
int offset,
@@ -194,8 +89,7 @@ internal static class UnpackUtility
long M,
N;
new Span<int>(dec.DecodeNum).Clear(); // memset(Dec->DecodeNum,0,Size*sizeof(*Dec->DecodeNum));
new Span<int>(dec.DecodeNum).Clear();
for (i = 0; i < size; i++)
{
lenCount[lenTab[offset + i] & 0xF]++;

View File

@@ -1,3 +1,4 @@
#if NETSTANDARD2_0 || NETFRAMEWORK
using System;
using System.IO;
using SharpCompress.Common.Rar.Headers;
@@ -106,3 +107,4 @@ internal partial class Unpack : IRarUnpack
public static byte[] EnsureCapacity(byte[] array, int length) =>
array.Length < length ? new byte[length] : array;
}
#endif

View File

@@ -413,7 +413,7 @@ internal partial class Unpack
else
//x memcpy(Mem,Window+BlockStart,BlockLength);
{
Utility.Copy(Window, BlockStart, Mem, 0, BlockLength);
Buffer.BlockCopy(Window, (int)BlockStart, Mem, 0, (int)BlockLength);
}
}
else
@@ -427,9 +427,21 @@ internal partial class Unpack
else
{
//x memcpy(Mem,Window+BlockStart,FirstPartLength);
Utility.Copy(Window, BlockStart, Mem, 0, FirstPartLength);
Buffer.BlockCopy(
Window,
(int)BlockStart,
Mem,
0,
(int)FirstPartLength
);
//x memcpy(Mem+FirstPartLength,Window,BlockEnd);
Utility.Copy(Window, 0, Mem, FirstPartLength, BlockEnd);
Buffer.BlockCopy(
Window,
0,
Mem,
(int)FirstPartLength,
(int)BlockEnd
);
}
}

View File

@@ -1,3 +1,4 @@
#if NETSTANDARD2_0 || NETFRAMEWORK
#nullable disable
using System;
@@ -29,12 +30,12 @@ internal sealed partial class Unpack : BitInput
Suspended = false;
UnpAllBuf = false;
UnpSomeRead = false;
/*#if RarV2017_RAR_SMP
MaxUserThreads = 1;
UnpThreadPool = CreateThreadPool();
ReadBufMT = null;
UnpThreadData = null;
#endif*/
// #if RarV2017_RAR_SMP
// MaxUserThreads = 1;
// UnpThreadPool = CreateThreadPool();
// ReadBufMT = null;
// UnpThreadData = null;
// #endif
MaxWinSize = 0;
MaxWinMask = 0;
@@ -197,21 +198,21 @@ internal sealed partial class Unpack : BitInput
break;
#endif
case 50: // RAR 5.0 compression algorithm.
/*#if RarV2017_RAR_SMP
if (MaxUserThreads > 1)
{
// We do not use the multithreaded unpack routine to repack RAR archives
// in 'suspended' mode, because unlike the single threaded code it can
// write more than one dictionary for same loop pass. So we would need
// larger buffers of unknown size. Also we do not support multithreading
// in fragmented window mode.
if (!Fragmented)
{
Unpack5MT(Solid);
break;
}
}
#endif*/
// #if RarV2017_RAR_SMP
// if (MaxUserThreads > 1)
// {
// // We do not use the multithreaded unpack routine to repack RAR archives
// // in 'suspended' mode, because unlike the single threaded code it can
// // write more than one dictionary for same loop pass. So we would need
// // larger buffers of unknown size. Also we do not support multithreading
// // in fragmented window mode.
// if (!Fragmented)
// {
// Unpack5MT(Solid);
// break;
// }
// }
// #endif
Unpack5(Solid);
break;
#if !Rar2017_NOSTRICT
@@ -407,3 +408,4 @@ internal sealed partial class Unpack : BitInput
}
}
}
#endif

View File

@@ -0,0 +1,411 @@
#if !NETSTANDARD2_0 && !NETFRAMEWORK
#nullable disable
using System;
using SharpCompress.Common;
using static SharpCompress.Compressors.Rar.UnpackV2017.PackDef;
using static SharpCompress.Compressors.Rar.UnpackV2017.UnpackGlobal;
#if !Rar2017_64bit
using size_t = System.UInt32;
#else
using nint = System.Int64;
using nuint = System.UInt64;
using size_t = System.UInt64;
#endif
namespace SharpCompress.Compressors.Rar.UnpackV2017;
internal sealed partial class Unpack : BitInput
{
public Unpack( /* ComprDataIO *DataIO */
)
//:Inp(true),VMCodeInp(true)
: base(true)
{
_UnpackCtor();
//UnpIO=DataIO;
Window = null;
Fragmented = false;
Suspended = false;
UnpAllBuf = false;
UnpSomeRead = false;
// #if RarV2017_RAR_SMP
// MaxUserThreads = 1;
// UnpThreadPool = CreateThreadPool();
// ReadBufMT = null;
// UnpThreadData = null;
// #endif
MaxWinSize = 0;
MaxWinMask = 0;
// Perform initialization, which should be done only once for all files.
// It prevents crash if first DoUnpack call is later made with wrong
// (true) 'Solid' value.
UnpInitData(false);
#if !RarV2017_SFX_MODULE
// RAR 1.5 decompression initialization
UnpInitData15(false);
InitHuff();
#endif
}
// later: may need Dispose() if we support thread pool
//Unpack::~Unpack()
//{
// InitFilters30(false);
//
// if (Window!=null)
// free(Window);
//#if RarV2017_RAR_SMP
// DestroyThreadPool(UnpThreadPool);
// delete[] ReadBufMT;
// delete[] UnpThreadData;
//#endif
//}
private void Init(size_t WinSize, bool Solid)
{
// If 32-bit RAR unpacks an archive with 4 GB dictionary, the window size
// will be 0 because of size_t overflow. Let's issue the memory error.
if (WinSize == 0)
//ErrHandler.MemoryError();
{
throw new InvalidFormatException(
"invalid window size (possibly due to a rar file with a 4GB being unpacked on a 32-bit platform)"
);
}
// Minimum window size must be at least twice more than maximum possible
// size of filter block, which is 0x10000 in RAR now. If window size is
// smaller, we can have a block with never cleared flt->NextWindow flag
// in UnpWriteBuf(). Minimum window size 0x20000 would be enough, but let's
// use 0x40000 for extra safety and possible filter area size expansion.
const size_t MinAllocSize = 0x40000;
if (WinSize < MinAllocSize)
{
WinSize = MinAllocSize;
}
if (WinSize <= MaxWinSize) // Use the already allocated window.
{
return;
}
if ((WinSize >> 16) > 0x10000) // Window size must not exceed 4 GB.
{
return;
}
// Archiving code guarantees that window size does not grow in the same
// solid stream. So if we are here, we are either creating a new window
// or increasing the size of non-solid window. So we could safely reject
// current window data without copying them to a new window, though being
// extra cautious, we still handle the solid window grow case below.
var Grow = Solid && (Window != null || Fragmented);
// We do not handle growth for existing fragmented window.
if (Grow && Fragmented)
//throw std::bad_alloc();
{
throw new InvalidFormatException("Grow && Fragmented");
}
var NewWindow = Fragmented ? null : new byte[WinSize];
if (NewWindow == null)
{
if (Grow || WinSize < 0x1000000)
{
// We do not support growth for new fragmented window.
// Also exclude RAR4 and small dictionaries.
//throw std::bad_alloc();
throw new InvalidFormatException("Grow || WinSize<0x1000000");
}
else
{
if (Window != null) // If allocated by preceding files.
{
//free(Window);
Window = null;
}
FragWindow.Init(WinSize);
Fragmented = true;
}
}
if (!Fragmented)
{
// Clean the window to generate the same output when unpacking corrupt
// RAR files, which may access unused areas of sliding dictionary.
// sharpcompress: don't need this, freshly allocated above
//memset(NewWindow,0,WinSize);
// If Window is not NULL, it means that window size has grown.
// In solid streams we need to copy data to a new window in such case.
// RAR archiving code does not allow it in solid streams now,
// but let's implement it anyway just in case we'll change it sometimes.
if (Grow)
{
for (size_t I = 1; I <= MaxWinSize; I++)
{
NewWindow[(UnpPtr - I) & (WinSize - 1)] = Window[
(UnpPtr - I) & (MaxWinSize - 1)
];
}
}
//if (Window!=null)
// free(Window);
Window = NewWindow;
}
MaxWinSize = WinSize;
MaxWinMask = MaxWinSize - 1;
}
private void DoUnpack(uint Method, bool Solid)
{
// Methods <50 will crash in Fragmented mode when accessing NULL Window.
// They cannot be called in such mode now, but we check it below anyway
// just for extra safety.
switch (Method)
{
#if !RarV2017_SFX_MODULE
case 15: // rar 1.5 compression
if (!Fragmented)
{
Unpack15(Solid);
}
break;
case 20: // rar 2.x compression
case 26: // files larger than 2GB
if (!Fragmented)
{
Unpack20(Solid);
}
break;
#endif
#if !RarV2017_RAR5ONLY
case 29: // rar 3.x compression
if (!Fragmented)
{
throw new NotImplementedException();
}
break;
#endif
case 50: // RAR 5.0 compression algorithm.
// #if RarV2017_RAR_SMP
// if (MaxUserThreads > 1)
// {
// // We do not use the multithreaded unpack routine to repack RAR archives
// // in 'suspended' mode, because unlike the single threaded code it can
// // write more than one dictionary for same loop pass. So we would need
// // larger buffers of unknown size. Also we do not support multithreading
// // in fragmented window mode.
// if (!Fragmented)
// {
// Unpack5MT(Solid);
// break;
// }
// }
// #endif
Unpack5(Solid);
break;
#if !Rar2017_NOSTRICT
default:
throw new InvalidFormatException("unknown compression method " + Method);
#endif
}
}
private void UnpInitData(bool Solid)
{
if (!Solid)
{
new Span<uint>(OldDist).Clear();
OldDistPtr = 0;
LastDist = LastLength = 0;
// memset(Window,0,MaxWinSize);
//memset(&BlockTables,0,sizeof(BlockTables));
BlockTables = new UnpackBlockTables();
// sharpcompress: no default ctor for struct
BlockTables.Init();
UnpPtr = WrPtr = 0;
WriteBorder = Math.Min(MaxWinSize, UNPACK_MAX_WRITE) & MaxWinMask;
}
// Filters never share several solid files, so we can safely reset them
// even in solid archive.
InitFilters();
Inp.InitBitInput();
WrittenFileSize = 0;
ReadTop = 0;
ReadBorder = 0;
//memset(&BlockHeader,0,sizeof(BlockHeader));
BlockHeader = new UnpackBlockHeader();
BlockHeader.BlockSize = -1; // '-1' means not defined yet.
#if !RarV2017_SFX_MODULE
UnpInitData20(Solid);
#endif
//UnpInitData30(Solid);
UnpInitData50(Solid);
}
// LengthTable contains the length in bits for every element of alphabet.
// Dec is the structure to decode Huffman code/
// Size is size of length table and DecodeNum field in Dec structure,
private void MakeDecodeTables(Span<byte> LengthTable, int offset, DecodeTable Dec, uint Size)
{
// Size of alphabet and DecodePos array.
Dec.MaxNum = Size;
// Calculate how many entries for every bit length in LengthTable we have.
var LengthCount = new uint[16];
//memset(LengthCount,0,sizeof(LengthCount));
for (size_t I = 0; I < Size; I++)
{
LengthCount[LengthTable[checked((int)(offset + I))] & 0xf]++;
}
// We must not calculate the number of zero length codes.
LengthCount[0] = 0;
// Set the entire DecodeNum to zero.
//memset(Dec->DecodeNum,0,Size*sizeof(*Dec->DecodeNum));
new Span<ushort>(Dec.DecodeNum).Clear();
// Initialize not really used entry for zero length code.
Dec.DecodePos[0] = 0;
// Start code for bit length 1 is 0.
Dec.DecodeLen[0] = 0;
// Right aligned upper limit code for current bit length.
uint UpperLimit = 0;
for (var I = 1; I < 16; I++)
{
// Adjust the upper limit code.
UpperLimit += LengthCount[I];
// Left aligned upper limit code.
var LeftAligned = UpperLimit << (16 - I);
// Prepare the upper limit code for next bit length.
UpperLimit *= 2;
// Store the left aligned upper limit code.
Dec.DecodeLen[I] = LeftAligned;
// Every item of this array contains the sum of all preceding items.
// So it contains the start position in code list for every bit length.
Dec.DecodePos[I] = Dec.DecodePos[I - 1] + LengthCount[I - 1];
}
// Prepare the copy of DecodePos. We'll modify this copy below,
// so we cannot use the original DecodePos.
var CopyDecodePos = new uint[Dec.DecodePos.Length];
//memcpy(CopyDecodePos,Dec->DecodePos,sizeof(CopyDecodePos));
Array.Copy(Dec.DecodePos, CopyDecodePos, CopyDecodePos.Length);
// For every bit length in the bit length table and so for every item
// of alphabet.
for (uint I = 0; I < Size; I++)
{
// Get the current bit length.
var _CurBitLength = (byte)(LengthTable[checked((int)(offset + I))] & 0xf);
if (_CurBitLength != 0)
{
// Last position in code list for current bit length.
var LastPos = CopyDecodePos[_CurBitLength];
// Prepare the decode table, so this position in code list will be
// decoded to current alphabet item number.
Dec.DecodeNum[LastPos] = (ushort)I;
// We'll use next position number for this bit length next time.
// So we pass through the entire range of positions available
// for every bit length.
CopyDecodePos[_CurBitLength]++;
}
}
// Define the number of bits to process in quick mode. We use more bits
// for larger alphabets. More bits means that more codes will be processed
// in quick mode, but also that more time will be spent to preparation
// of tables for quick decode.
switch (Size)
{
case NC:
case NC20:
case NC30:
Dec.QuickBits = MAX_QUICK_DECODE_BITS;
break;
default:
Dec.QuickBits = MAX_QUICK_DECODE_BITS - 3;
break;
}
// Size of tables for quick mode.
var QuickDataSize = 1U << (int)Dec.QuickBits;
// Bit length for current code, start from 1 bit codes. It is important
// to use 1 bit instead of 0 for minimum code length, so we are moving
// forward even when processing a corrupt archive.
//uint CurBitLength=1;
byte CurBitLength = 1;
// For every right aligned bit string which supports the quick decoding.
for (uint Code = 0; Code < QuickDataSize; Code++)
{
// Left align the current code, so it will be in usual bit field format.
var BitField = Code << (int)(16 - Dec.QuickBits);
// Prepare the table for quick decoding of bit lengths.
// Find the upper limit for current bit field and adjust the bit length
// accordingly if necessary.
while (CurBitLength < Dec.DecodeLen.Length && BitField >= Dec.DecodeLen[CurBitLength])
{
CurBitLength++;
}
// Translation of right aligned bit string to bit length.
Dec.QuickLen[Code] = CurBitLength;
// Prepare the table for quick translation of position in code list
// to position in alphabet.
// Calculate the distance from the start code for current bit length.
var Dist = BitField - Dec.DecodeLen[CurBitLength - 1];
// Right align the distance.
Dist >>= (16 - CurBitLength);
// Now we can calculate the position in the code list. It is the sum
// of first position for current bit length and right aligned distance
// between our bit field and start code for current bit length.
uint Pos;
if (
CurBitLength < Dec.DecodePos.Length
&& (Pos = Dec.DecodePos[CurBitLength] + Dist) < Size
)
{
// Define the code to alphabet number translation.
Dec.QuickNum[Code] = Dec.DecodeNum[Pos];
}
else
{
// Can be here for length table filled with zeroes only (empty).
Dec.QuickNum[Code] = 0;
}
}
}
}
#endif

View File

@@ -0,0 +1,115 @@
#if !NETSTANDARD2_0 && !NETFRAMEWORK
using System;
using System.Buffers;
using System.IO;
using System.Threading.Tasks;
using SharpCompress.Common.Rar.Headers;
#if !Rar2017_64bit
using size_t = System.UInt32;
#else
using nint = System.Int64;
using nuint = System.UInt64;
using size_t = System.UInt64;
#endif
namespace SharpCompress.Compressors.Rar.UnpackV2017;
internal partial class Unpack : IRarUnpack
{
private FileHeader fileHeader;
private Stream readStream;
private Stream writeStream;
private void _UnpackCtor()
{
for (var i = 0; i < AudV.Length; i++)
{
AudV[i] = new AudioVariables();
}
}
private int UnpIO_UnpRead(byte[] buf, int offset, int count) =>
// NOTE: caller has logic to check for -1 for error we throw instead.
readStream.Read(buf, offset, count);
private void UnpIO_UnpWrite(byte[] buf, size_t offset, uint count) =>
writeStream.Write(buf, checked((int)offset), checked((int)count));
public ValueTask DoUnpackAsync(FileHeader fileHeader, Stream readStream, Stream writeStream)
{
// as of 12/2017 .NET limits array indexing to using a signed integer
// MaxWinSize causes unpack to use a fragmented window when the file
// window size exceeds MaxWinSize
// uggh, that's not how this variable is used, it's the size of the currently allocated window buffer
//x MaxWinSize = ((uint)int.MaxValue) + 1;
// may be long.MaxValue which could indicate unknown size (not present in header)
DestUnpSize = fileHeader.UncompressedSize;
this.fileHeader = fileHeader;
this.readStream = readStream;
this.writeStream = writeStream;
if (!fileHeader.IsStored)
{
Init(fileHeader.WindowSize, fileHeader.IsSolid);
}
Suspended = false;
return DoUnpackAsync();
}
public ValueTask DoUnpackAsync()
{
if (fileHeader.IsStored)
{
return UnstoreFileAsync();
}
else
{
DoUnpack(fileHeader.CompressionAlgorithm, fileHeader.IsSolid);
return new ValueTask();
}
}
private async ValueTask UnstoreFileAsync()
{
var length = (int)Math.Min(0x10000, DestUnpSize);
var buffer = ArrayPool<byte>.Shared.Rent(length);
do
{
var memory = new Memory<byte>(buffer, 0, length);
var n = await readStream.ReadAsync(memory);
if (n == 0)
{
break;
}
await writeStream.WriteAsync(memory.Slice(0, n));
DestUnpSize -= n;
} while (!Suspended);
}
public bool Suspended { get; set; }
public long DestSize => DestUnpSize;
public int Char
{
get
{
// TODO: coderb: not sure where the "MAXSIZE-30" comes from, ported from V1 code
if (InAddr > MAX_SIZE - 30)
{
UnpReadBuf();
}
return InBuf[InAddr++];
}
}
public int PpmEscChar
{
get => PPMEscChar;
set => PPMEscChar = value;
}
public static byte[] EnsureCapacity(byte[] array, int length) =>
array.Length < length ? new byte[length] : array;
}
#endif

View File

@@ -1,6 +1,9 @@
using System;
using System.Buffers;
namespace SharpCompress.Compressors.Rar.VM;
internal class BitInput
internal class BitInput : IDisposable
{
/// <summary> the max size of the input</summary>
internal const int MAX_SIZE = 0x8000;
@@ -20,9 +23,11 @@ internal class BitInput
set => inBit = value;
}
public bool ExternalBuffer;
private byte[] _privateBuffer = ArrayPool<byte>.Shared.Rent(MAX_SIZE);
private bool _disposed;
/// <summary> </summary>
internal BitInput() => InBuf = new byte[MAX_SIZE];
internal BitInput() => InBuf = _privateBuffer;
internal byte[] InBuf { get; }
@@ -87,4 +92,14 @@ internal class BitInput
/// <returns> true if an Oververflow would occur
/// </returns>
internal bool Overflow(int IncPtr) => (inAddr + IncPtr >= MAX_SIZE);
public virtual void Dispose()
{
if (_disposed)
{
return;
}
ArrayPool<byte>.Shared.Return(_privateBuffer);
_disposed = true;
}
}

View File

@@ -1,6 +1,5 @@
#nullable disable
using System;
using System.Buffers;
using System.Buffers.Binary;
using System.Collections.Generic;
@@ -16,7 +15,9 @@ internal sealed class RarVM : BitInput
// Mem.set_Renamed(offset + 3, Byte.valueOf((sbyte) ((Utility.URShift(value_Renamed, 24)) & 0xff)));
//}
internal byte[] Mem { get; private set; }
internal byte[] Mem => _memory.NotNull();
private byte[]? _memory = ArrayPool<byte>.Shared.Rent(VM_MEMSIZE + 4);
public const int VM_MEMSIZE = 0x40000;
@@ -40,11 +41,18 @@ internal sealed class RarVM : BitInput
private int IP;
internal RarVM() =>
//InitBlock();
Mem = null;
internal RarVM() { }
internal void init() => Mem ??= new byte[VM_MEMSIZE + 4];
public override void Dispose()
{
base.Dispose();
if (_memory is null)
{
return;
}
ArrayPool<byte>.Shared.Return(_memory);
_memory = null;
}
private bool IsVMMem(byte[] mem) => Mem == mem;
@@ -776,9 +784,10 @@ internal sealed class RarVM : BitInput
}
}
public void prepare(ReadOnlySpan<byte> code, int codeSize, VMPreparedProgram prg)
public void prepare(ReadOnlySpan<byte> code, VMPreparedProgram prg)
{
InitBitInput();
var codeSize = code.Length;
var cpLength = Math.Min(MAX_SIZE, codeSize);
// memcpy(inBuf,Code,Min(CodeSize,BitInput::MAX_SIZE));
@@ -795,7 +804,7 @@ internal sealed class RarVM : BitInput
prg.CommandCount = 0;
if (xorSum == code[0])
{
var filterType = IsStandardFilter(code, codeSize);
var filterType = IsStandardFilter(code);
if (filterType != VMStandardFilters.VMSF_NONE)
{
var curCmd = new VMPreparedCommand();
@@ -1105,7 +1114,7 @@ internal sealed class RarVM : BitInput
}
}
private VMStandardFilters IsStandardFilter(ReadOnlySpan<byte> code, int codeSize)
private VMStandardFilters IsStandardFilter(ReadOnlySpan<byte> code)
{
VMStandardFilterSignature[] stdList =
{
@@ -1130,6 +1139,7 @@ internal sealed class RarVM : BitInput
private void ExecuteStandardFilter(VMStandardFilters filterType)
{
var mem = Mem;
switch (filterType)
{
case VMStandardFilters.VMSF_E8:
@@ -1148,7 +1158,7 @@ internal sealed class RarVM : BitInput
);
for (var curPos = 0; curPos < dataSize - 4; )
{
var curByte = Mem[curPos++];
var curByte = mem[curPos++];
if (curByte == 0xe8 || curByte == cmpByte2)
{
// #ifdef PRESENT_INT32
@@ -1164,19 +1174,19 @@ internal sealed class RarVM : BitInput
// SET_VALUE(false,Data,Addr-Offset);
// #else
var offset = curPos + fileOffset;
long Addr = GetValue(false, Mem, curPos);
long Addr = GetValue(false, mem, curPos);
if ((Addr & unchecked((int)0x80000000)) != 0)
{
if (((Addr + offset) & unchecked((int)0x80000000)) == 0)
{
SetValue(false, Mem, curPos, (int)Addr + fileSize);
SetValue(false, mem, curPos, (int)Addr + fileSize);
}
}
else
{
if (((Addr - fileSize) & unchecked((int)0x80000000)) != 0)
{
SetValue(false, Mem, curPos, (int)(Addr - offset));
SetValue(false, mem, curPos, (int)(Addr - offset));
}
}
@@ -1204,7 +1214,7 @@ internal sealed class RarVM : BitInput
while (curPos < dataSize - 21)
{
var Byte = (Mem[curPos] & 0x1f) - 0x10;
var Byte = (mem[curPos] & 0x1f) - 0x10;
if (Byte >= 0)
{
var cmdMask = Masks[Byte];
@@ -1250,7 +1260,7 @@ internal sealed class RarVM : BitInput
var channels = R[0] & unchecked((int)0xFFffFFff);
var srcPos = 0;
var border = (dataSize * 2) & unchecked((int)0xFFffFFff);
SetValue(false, Mem, VM_GLOBALMEMADDR + 0x20, dataSize);
SetValue(false, mem, VM_GLOBALMEMADDR + 0x20, dataSize);
if (dataSize >= VM_GLOBALMEMADDR / 2)
{
break;
@@ -1268,7 +1278,7 @@ internal sealed class RarVM : BitInput
destPos += channels
)
{
Mem[destPos] = (PrevByte = (byte)(PrevByte - Mem[srcPos++]));
mem[destPos] = (PrevByte = (byte)(PrevByte - mem[srcPos++]));
}
}
}
@@ -1283,7 +1293,7 @@ internal sealed class RarVM : BitInput
var channels = 3;
var srcPos = 0;
var destDataPos = dataSize;
SetValue(false, Mem, VM_GLOBALMEMADDR + 0x20, dataSize);
SetValue(false, mem, VM_GLOBALMEMADDR + 0x20, dataSize);
if (dataSize >= VM_GLOBALMEMADDR / 2 || posR < 0)
{
break;
@@ -1299,8 +1309,8 @@ internal sealed class RarVM : BitInput
if (upperPos >= 3)
{
var upperDataPos = destDataPos + upperPos;
var upperByte = Mem[upperDataPos] & 0xff;
var upperLeftByte = Mem[upperDataPos - 3] & 0xff;
var upperByte = mem[upperDataPos] & 0xff;
var upperLeftByte = mem[upperDataPos - 3] & 0xff;
predicted = prevByte + upperByte - upperLeftByte;
var pa = Math.Abs((int)(predicted - prevByte));
var pb = Math.Abs((int)(predicted - upperByte));
@@ -1326,15 +1336,15 @@ internal sealed class RarVM : BitInput
predicted = prevByte;
}
prevByte = ((predicted - Mem[srcPos++]) & 0xff) & 0xff;
Mem[destDataPos + i] = (byte)(prevByte & 0xff);
prevByte = ((predicted - mem[srcPos++]) & 0xff) & 0xff;
mem[destDataPos + i] = (byte)(prevByte & 0xff);
}
}
for (int i = posR, border = dataSize - 2; i < border; i += 3)
{
var G = Mem[destDataPos + i + 1];
Mem[destDataPos + i] = (byte)(Mem[destDataPos + i] + G);
Mem[destDataPos + i + 2] = (byte)(Mem[destDataPos + i + 2] + G);
var G = mem[destDataPos + i + 1];
mem[destDataPos + i] = (byte)(mem[destDataPos + i] + G);
mem[destDataPos + i + 2] = (byte)(mem[destDataPos + i + 2] + G);
}
}
break;
@@ -1347,7 +1357,7 @@ internal sealed class RarVM : BitInput
var destDataPos = dataSize;
//byte *SrcData=Mem,*DestData=SrcData+DataSize;
SetValue(false, Mem, VM_GLOBALMEMADDR + 0x20, dataSize);
SetValue(false, mem, VM_GLOBALMEMADDR + 0x20, dataSize);
if (dataSize >= VM_GLOBALMEMADDR / 2)
{
break;
@@ -1377,10 +1387,10 @@ internal sealed class RarVM : BitInput
var predicted = (8 * prevByte) + (K1 * D1) + (K2 * D2) + (K3 * D3);
predicted = Utility.URShift(predicted, 3) & 0xff;
long curByte = Mem[srcPos++];
long curByte = mem[srcPos++];
predicted -= curByte;
Mem[destDataPos + i] = (byte)predicted;
mem[destDataPos + i] = (byte)predicted;
prevDelta = (byte)(predicted - prevByte);
//fix java byte
@@ -1480,15 +1490,15 @@ internal sealed class RarVM : BitInput
}
while (srcPos < dataSize)
{
var curByte = Mem[srcPos++];
if (curByte == 2 && (curByte = Mem[srcPos++]) != 2)
var curByte = mem[srcPos++];
if (curByte == 2 && (curByte = mem[srcPos++]) != 2)
{
curByte = (byte)(curByte - 32);
}
Mem[destPos++] = curByte;
mem[destPos++] = curByte;
}
SetValue(false, Mem, VM_GLOBALMEMADDR + 0x1c, destPos - dataSize);
SetValue(false, Mem, VM_GLOBALMEMADDR + 0x20, dataSize);
SetValue(false, mem, VM_GLOBALMEMADDR + 0x1c, destPos - dataSize);
SetValue(false, mem, VM_GLOBALMEMADDR + 0x20, dataSize);
}
break;
}
@@ -1528,15 +1538,14 @@ internal sealed class RarVM : BitInput
{
if (pos < VM_MEMSIZE)
{
//&& data!=Mem+Pos)
//memmove(Mem+Pos,Data,Min(DataSize,VM_MEMSIZE-Pos));
for (var i = 0; i < Math.Min(data.Length - offset, dataSize); i++)
// Use Array.Copy for fast bulk memory operations instead of byte-by-byte loop
// Calculate how much data can actually fit in VM memory
int copyLength = Math.Min(dataSize, VM_MEMSIZE - pos);
copyLength = Math.Min(copyLength, data.Length - offset);
if (copyLength > 0)
{
if ((VM_MEMSIZE - pos) < i)
{
break;
}
Mem[pos + i] = data[offset + i];
Array.Copy(data, offset, Mem, pos, copyLength);
}
}
}

View File

@@ -1,78 +1,79 @@
namespace SharpCompress.Compressors.Shrink;
internal class BitStream
namespace SharpCompress.Compressors.Shrink
{
private byte[] _src;
private int _srcLen;
private int _byteIdx;
private int _bitIdx;
private int _bitsLeft;
private ulong _bitBuffer;
private static uint[] _maskBits = new uint[17]
internal class BitStream
{
0U,
1U,
3U,
7U,
15U,
31U,
63U,
(uint)sbyte.MaxValue,
(uint)byte.MaxValue,
511U,
1023U,
2047U,
4095U,
8191U,
16383U,
(uint)short.MaxValue,
(uint)ushort.MaxValue,
};
public BitStream(byte[] src, int srcLen)
{
_src = src;
_srcLen = srcLen;
_byteIdx = 0;
_bitIdx = 0;
}
public int BytesRead => (_byteIdx << 3) + _bitIdx;
private int NextByte()
{
if (_byteIdx >= _srcLen)
private byte[] _src;
private int _srcLen;
private int _byteIdx;
private int _bitIdx;
private int _bitsLeft;
private ulong _bitBuffer;
private static uint[] _maskBits = new uint[17]
{
return 0;
0U,
1U,
3U,
7U,
15U,
31U,
63U,
(uint)sbyte.MaxValue,
(uint)byte.MaxValue,
511U,
1023U,
2047U,
4095U,
8191U,
16383U,
(uint)short.MaxValue,
(uint)ushort.MaxValue,
};
public BitStream(byte[] src, int srcLen)
{
_src = src;
_srcLen = srcLen;
_byteIdx = 0;
_bitIdx = 0;
}
return _src[_byteIdx++];
}
public int BytesRead => (_byteIdx << 3) + _bitIdx;
public int NextBits(int nbits)
{
var result = 0;
if (nbits > _bitsLeft)
private int NextByte()
{
int num;
while (_bitsLeft <= 24 && (num = NextByte()) != 1234)
if (_byteIdx >= _srcLen)
{
_bitBuffer |= (ulong)num << _bitsLeft;
_bitsLeft += 8;
return 0;
}
}
result = (int)((long)_bitBuffer & (long)_maskBits[nbits]);
_bitBuffer >>= nbits;
_bitsLeft -= nbits;
return result;
}
public bool Advance(int count)
{
if (_byteIdx > _srcLen)
{
return false;
return _src[_byteIdx++];
}
public int NextBits(int nbits)
{
var result = 0;
if (nbits > _bitsLeft)
{
int num;
while (_bitsLeft <= 24 && (num = NextByte()) != 1234)
{
_bitBuffer |= (ulong)num << _bitsLeft;
_bitsLeft += 8;
}
}
result = (int)((long)_bitBuffer & (long)_maskBits[nbits]);
_bitBuffer >>= nbits;
_bitsLeft -= nbits;
return result;
}
public bool Advance(int count)
{
if (_byteIdx > _srcLen)
{
return false;
}
return true;
}
return true;
}
}

View File

@@ -1,297 +1,275 @@
using System;
namespace SharpCompress.Compressors.Shrink;
public class HwUnshrink
namespace SharpCompress.Compressors.Shrink
{
private const int MIN_CODE_SIZE = 9;
private const int MAX_CODE_SIZE = 13;
private const ushort MAX_CODE = (ushort)((1U << MAX_CODE_SIZE) - 1);
private const ushort INVALID_CODE = ushort.MaxValue;
private const ushort CONTROL_CODE = 256;
private const ushort INC_CODE_SIZE = 1;
private const ushort PARTIAL_CLEAR = 2;
private const int HASH_BITS = MAX_CODE_SIZE + 1; // For a load factor of 0.5.
private const int HASHTAB_SIZE = 1 << HASH_BITS;
private const ushort UNKNOWN_LEN = ushort.MaxValue;
private struct CodeTabEntry
public class HwUnshrink
{
public int prefixCode; // INVALID_CODE means the entry is invalid.
public byte extByte;
public ushort len;
public int lastDstPos;
}
private const int MIN_CODE_SIZE = 9;
private const int MAX_CODE_SIZE = 13;
private static void CodeTabInit(CodeTabEntry[] codeTab)
{
for (var i = 0; i <= byte.MaxValue; i++)
private const ushort MAX_CODE = (ushort)((1U << MAX_CODE_SIZE) - 1);
private const ushort INVALID_CODE = ushort.MaxValue;
private const ushort CONTROL_CODE = 256;
private const ushort INC_CODE_SIZE = 1;
private const ushort PARTIAL_CLEAR = 2;
private const int HASH_BITS = MAX_CODE_SIZE + 1; // For a load factor of 0.5.
private const int HASHTAB_SIZE = 1 << HASH_BITS;
private const ushort UNKNOWN_LEN = ushort.MaxValue;
private struct CodeTabEntry
{
codeTab[i].prefixCode = (ushort)i;
codeTab[i].extByte = (byte)i;
codeTab[i].len = 1;
public int prefixCode; // INVALID_CODE means the entry is invalid.
public byte extByte;
public ushort len;
public int lastDstPos;
}
for (var i = byte.MaxValue + 1; i <= MAX_CODE; i++)
private static void CodeTabInit(CodeTabEntry[] codeTab)
{
codeTab[i].prefixCode = INVALID_CODE;
}
}
private static void UnshrinkPartialClear(CodeTabEntry[] codeTab, ref CodeQueue queue)
{
var isPrefix = new bool[MAX_CODE + 1];
int codeQueueSize;
// Scan for codes that have been used as a prefix.
for (var i = CONTROL_CODE + 1; i <= MAX_CODE; i++)
{
if (codeTab[i].prefixCode != INVALID_CODE)
for (var i = 0; i <= byte.MaxValue; i++)
{
isPrefix[codeTab[i].prefixCode] = true;
codeTab[i].prefixCode = (ushort)i;
codeTab[i].extByte = (byte)i;
codeTab[i].len = 1;
}
}
// Clear "non-prefix" codes in the table; populate the code queue.
codeQueueSize = 0;
for (var i = CONTROL_CODE + 1; i <= MAX_CODE; i++)
{
if (!isPrefix[i])
for (var i = byte.MaxValue + 1; i <= MAX_CODE; i++)
{
codeTab[i].prefixCode = INVALID_CODE;
queue.codes[codeQueueSize++] = (ushort)i;
}
}
queue.codes[codeQueueSize] = INVALID_CODE; // End-of-queue marker.
queue.nextIdx = 0;
}
private static bool ReadCode(
BitStream stream,
ref int codeSize,
CodeTabEntry[] codeTab,
ref CodeQueue queue,
out int nextCode
)
{
int code,
controlCode;
code = (int)stream.NextBits(codeSize);
if (!stream.Advance(codeSize))
private static void UnshrinkPartialClear(CodeTabEntry[] codeTab, ref CodeQueue queue)
{
nextCode = INVALID_CODE;
return false;
var isPrefix = new bool[MAX_CODE + 1];
int codeQueueSize;
// Scan for codes that have been used as a prefix.
for (var i = CONTROL_CODE + 1; i <= MAX_CODE; i++)
{
if (codeTab[i].prefixCode != INVALID_CODE)
{
isPrefix[codeTab[i].prefixCode] = true;
}
}
// Clear "non-prefix" codes in the table; populate the code queue.
codeQueueSize = 0;
for (var i = CONTROL_CODE + 1; i <= MAX_CODE; i++)
{
if (!isPrefix[i])
{
codeTab[i].prefixCode = INVALID_CODE;
queue.codes[codeQueueSize++] = (ushort)i;
}
}
queue.codes[codeQueueSize] = INVALID_CODE; // End-of-queue marker.
queue.nextIdx = 0;
}
// Handle regular codes (the common case).
if (code != CONTROL_CODE)
private static bool ReadCode(
BitStream stream,
ref int codeSize,
CodeTabEntry[] codeTab,
ref CodeQueue queue,
out int nextCode
)
{
nextCode = code;
return true;
}
int code,
controlCode;
code = (int)stream.NextBits(codeSize);
if (!stream.Advance(codeSize))
{
nextCode = INVALID_CODE;
return false;
}
// Handle regular codes (the common case).
if (code != CONTROL_CODE)
{
nextCode = code;
return true;
}
// Handle control codes.
controlCode = (ushort)stream.NextBits(codeSize);
if (!stream.Advance(codeSize))
{
nextCode = INVALID_CODE;
return true;
}
if (controlCode == INC_CODE_SIZE && codeSize < MAX_CODE_SIZE)
{
codeSize++;
return ReadCode(stream, ref codeSize, codeTab, ref queue, out nextCode);
}
if (controlCode == PARTIAL_CLEAR)
{
UnshrinkPartialClear(codeTab, ref queue);
return ReadCode(stream, ref codeSize, codeTab, ref queue, out nextCode);
}
// Handle control codes.
controlCode = (ushort)stream.NextBits(codeSize);
if (!stream.Advance(codeSize))
{
nextCode = INVALID_CODE;
return true;
}
if (controlCode == INC_CODE_SIZE && codeSize < MAX_CODE_SIZE)
private static void CopyFromPrevPos(byte[] dst, int prevPos, int dstPos, int len)
{
codeSize++;
return ReadCode(stream, ref codeSize, codeTab, ref queue, out nextCode);
if (dstPos + len > dst.Length)
{
// Not enough room in dst for the sloppy copy below.
Array.Copy(dst, prevPos, dst, dstPos, len);
return;
}
if (prevPos + len > dstPos)
{
// Benign one-byte overlap possible in the KwKwK case.
//assert(prevPos + len == dstPos + 1);
//assert(dst[prevPos] == dst[prevPos + len - 1]);
}
Buffer.BlockCopy(dst, prevPos, dst, dstPos, len);
}
if (controlCode == PARTIAL_CLEAR)
private static UnshrnkStatus OutputCode(
int code,
byte[] dst,
int dstPos,
int dstCap,
int prevCode,
CodeTabEntry[] codeTab,
ref CodeQueue queue,
out byte firstByte,
out int len
)
{
UnshrinkPartialClear(codeTab, ref queue);
return ReadCode(stream, ref codeSize, codeTab, ref queue, out nextCode);
}
int prefixCode;
nextCode = INVALID_CODE;
return true;
}
private static void CopyFromPrevPos(byte[] dst, int prevPos, int dstPos, int len)
{
if (dstPos + len > dst.Length)
{
// Not enough room in dst for the sloppy copy below.
Array.Copy(dst, prevPos, dst, dstPos, len);
return;
}
if (prevPos + len > dstPos)
{
// Benign one-byte overlap possible in the KwKwK case.
//assert(prevPos + len == dstPos + 1);
//assert(dst[prevPos] == dst[prevPos + len - 1]);
}
Buffer.BlockCopy(dst, prevPos, dst, dstPos, len);
}
private static UnshrnkStatus OutputCode(
int code,
byte[] dst,
int dstPos,
int dstCap,
int prevCode,
CodeTabEntry[] codeTab,
ref CodeQueue queue,
out byte firstByte,
out int len
)
{
int prefixCode;
//assert(code <= MAX_CODE && code != CONTROL_CODE);
//assert(dstPos < dstCap);
firstByte = 0;
if (code <= byte.MaxValue)
{
// Output literal byte.
firstByte = (byte)code;
len = 1;
dst[dstPos] = (byte)code;
return UnshrnkStatus.Ok;
}
if (codeTab[code].prefixCode == INVALID_CODE || codeTab[code].prefixCode == code)
{
// Reject invalid codes. Self-referential codes may exist in the table but cannot be used.
//assert(code <= MAX_CODE && code != CONTROL_CODE);
//assert(dstPos < dstCap);
firstByte = 0;
len = 0;
return UnshrnkStatus.Error;
}
if (code <= byte.MaxValue)
{
// Output literal byte.
firstByte = (byte)code;
len = 1;
dst[dstPos] = (byte)code;
return UnshrnkStatus.Ok;
}
if (codeTab[code].len != UNKNOWN_LEN)
{
// Output string with known length (the common case).
if (dstCap - dstPos < codeTab[code].len)
if (codeTab[code].prefixCode == INVALID_CODE || codeTab[code].prefixCode == code)
{
// Reject invalid codes. Self-referential codes may exist in the table but cannot be used.
firstByte = 0;
len = 0;
return UnshrnkStatus.Error;
}
if (codeTab[code].len != UNKNOWN_LEN)
{
// Output string with known length (the common case).
if (dstCap - dstPos < codeTab[code].len)
{
firstByte = 0;
len = 0;
return UnshrnkStatus.Full;
}
CopyFromPrevPos(dst, codeTab[code].lastDstPos, dstPos, codeTab[code].len);
firstByte = dst[dstPos];
len = codeTab[code].len;
return UnshrnkStatus.Ok;
}
// Output a string of unknown length.
//assert(codeTab[code].len == UNKNOWN_LEN);
prefixCode = codeTab[code].prefixCode;
// assert(prefixCode > CONTROL_CODE);
if (prefixCode == queue.codes[queue.nextIdx])
{
// The prefix code hasn't been added yet, but we were just about to: the KwKwK case.
//assert(codeTab[prevCode].prefixCode != INVALID_CODE);
codeTab[prefixCode].prefixCode = prevCode;
codeTab[prefixCode].extByte = firstByte;
codeTab[prefixCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[prefixCode].lastDstPos = codeTab[prevCode].lastDstPos;
dst[dstPos] = firstByte;
}
else if (codeTab[prefixCode].prefixCode == INVALID_CODE)
{
// The prefix code is still invalid.
firstByte = 0;
len = 0;
return UnshrnkStatus.Error;
}
// Output the prefix string, then the extension byte.
len = codeTab[prefixCode].len + 1;
if (dstCap - dstPos < len)
{
firstByte = 0;
len = 0;
return UnshrnkStatus.Full;
}
CopyFromPrevPos(dst, codeTab[code].lastDstPos, dstPos, codeTab[code].len);
CopyFromPrevPos(dst, codeTab[prefixCode].lastDstPos, dstPos, codeTab[prefixCode].len);
dst[dstPos + len - 1] = codeTab[code].extByte;
firstByte = dst[dstPos];
len = codeTab[code].len;
// Update the code table now that the string has a length and pos.
//assert(prevCode != code);
codeTab[code].len = (ushort)len;
codeTab[code].lastDstPos = dstPos;
return UnshrnkStatus.Ok;
}
// Output a string of unknown length.
//assert(codeTab[code].len == UNKNOWN_LEN);
prefixCode = codeTab[code].prefixCode;
// assert(prefixCode > CONTROL_CODE);
if (prefixCode == queue.codes[queue.nextIdx])
public static UnshrnkStatus Unshrink(
byte[] src,
int srcLen,
out int srcUsed,
byte[] dst,
int dstCap,
out int dstUsed
)
{
// The prefix code hasn't been added yet, but we were just about to: the KwKwK case.
//assert(codeTab[prevCode].prefixCode != INVALID_CODE);
codeTab[prefixCode].prefixCode = prevCode;
codeTab[prefixCode].extByte = firstByte;
codeTab[prefixCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[prefixCode].lastDstPos = codeTab[prevCode].lastDstPos;
dst[dstPos] = firstByte;
}
else if (codeTab[prefixCode].prefixCode == INVALID_CODE)
{
// The prefix code is still invalid.
firstByte = 0;
len = 0;
return UnshrnkStatus.Error;
}
var codeTab = new CodeTabEntry[HASHTAB_SIZE];
var queue = new CodeQueue();
var stream = new BitStream(src, srcLen);
int codeSize,
dstPos,
len;
int currCode,
prevCode,
newCode;
byte firstByte;
// Output the prefix string, then the extension byte.
len = codeTab[prefixCode].len + 1;
if (dstCap - dstPos < len)
{
firstByte = 0;
len = 0;
return UnshrnkStatus.Full;
}
CodeTabInit(codeTab);
CodeQueueInit(ref queue);
codeSize = MIN_CODE_SIZE;
dstPos = 0;
CopyFromPrevPos(dst, codeTab[prefixCode].lastDstPos, dstPos, codeTab[prefixCode].len);
dst[dstPos + len - 1] = codeTab[code].extByte;
firstByte = dst[dstPos];
// Update the code table now that the string has a length and pos.
//assert(prevCode != code);
codeTab[code].len = (ushort)len;
codeTab[code].lastDstPos = dstPos;
return UnshrnkStatus.Ok;
}
public static UnshrnkStatus Unshrink(
byte[] src,
int srcLen,
out int srcUsed,
byte[] dst,
int dstCap,
out int dstUsed
)
{
var codeTab = new CodeTabEntry[HASHTAB_SIZE];
var queue = new CodeQueue();
var stream = new BitStream(src, srcLen);
int codeSize,
dstPos,
len;
int currCode,
prevCode,
newCode;
byte firstByte;
CodeTabInit(codeTab);
CodeQueueInit(ref queue);
codeSize = MIN_CODE_SIZE;
dstPos = 0;
// Handle the first code separately since there is no previous code.
if (!ReadCode(stream, ref codeSize, codeTab, ref queue, out currCode))
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Ok;
}
//assert(currCode != CONTROL_CODE);
if (currCode > byte.MaxValue)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Error; // The first code must be a literal.
}
if (dstPos == dstCap)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Full;
}
firstByte = (byte)currCode;
dst[dstPos] = (byte)currCode;
codeTab[currCode].lastDstPos = dstPos;
dstPos++;
prevCode = currCode;
while (ReadCode(stream, ref codeSize, codeTab, ref queue, out currCode))
{
if (currCode == INVALID_CODE)
// Handle the first code separately since there is no previous code.
if (!ReadCode(stream, ref codeSize, codeTab, ref queue, out currCode))
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Error;
return UnshrnkStatus.Ok;
}
//assert(currCode != CONTROL_CODE);
if (currCode > byte.MaxValue)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Error; // The first code must be a literal.
}
if (dstPos == dstCap)
@@ -301,130 +279,153 @@ public class HwUnshrink
return UnshrnkStatus.Full;
}
// Handle KwKwK: next code used before being added.
if (currCode == queue.codes[queue.nextIdx])
firstByte = (byte)currCode;
dst[dstPos] = (byte)currCode;
codeTab[currCode].lastDstPos = dstPos;
dstPos++;
prevCode = currCode;
while (ReadCode(stream, ref codeSize, codeTab, ref queue, out currCode))
{
if (codeTab[prevCode].prefixCode == INVALID_CODE)
if (currCode == INVALID_CODE)
{
// The previous code is no longer valid.
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Error;
}
// Extend the previous code with its first byte.
//assert(currCode != prevCode);
codeTab[currCode].prefixCode = prevCode;
codeTab[currCode].extByte = firstByte;
codeTab[currCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[currCode].lastDstPos = codeTab[prevCode].lastDstPos;
//assert(dstPos < dstCap);
dst[dstPos] = firstByte;
}
// Output the string represented by the current code.
var status = OutputCode(
currCode,
dst,
dstPos,
dstCap,
prevCode,
codeTab,
ref queue,
out firstByte,
out len
);
if (status != UnshrnkStatus.Ok)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return status;
}
// Verify that the output matches walking the prefixes.
var c = currCode;
for (var i = 0; i < len; i++)
{
// assert(codeTab[c].len == len - i);
//assert(codeTab[c].extByte == dst[dstPos + len - i - 1]);
c = codeTab[c].prefixCode;
}
// Add a new code to the string table if there's room.
// The string is the previous code's string extended with the first byte of the current code's string.
newCode = CodeQueueRemoveNext(ref queue);
if (newCode != INVALID_CODE)
{
//assert(codeTab[prevCode].lastDstPos < dstPos);
codeTab[newCode].prefixCode = prevCode;
codeTab[newCode].extByte = firstByte;
codeTab[newCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[newCode].lastDstPos = codeTab[prevCode].lastDstPos;
if (codeTab[prevCode].prefixCode == INVALID_CODE)
if (dstPos == dstCap)
{
// prevCode was invalidated in a partial clearing. Until that code is re-used, the
// string represented by newCode is indeterminate.
codeTab[newCode].len = UNKNOWN_LEN;
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Full;
}
// If prevCode was invalidated in a partial clearing, it's possible that newCode == prevCode,
// in which case it will never be used or cleared.
// Handle KwKwK: next code used before being added.
if (currCode == queue.codes[queue.nextIdx])
{
if (codeTab[prevCode].prefixCode == INVALID_CODE)
{
// The previous code is no longer valid.
srcUsed = stream.BytesRead;
dstUsed = 0;
return UnshrnkStatus.Error;
}
// Extend the previous code with its first byte.
//assert(currCode != prevCode);
codeTab[currCode].prefixCode = prevCode;
codeTab[currCode].extByte = firstByte;
codeTab[currCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[currCode].lastDstPos = codeTab[prevCode].lastDstPos;
//assert(dstPos < dstCap);
dst[dstPos] = firstByte;
}
// Output the string represented by the current code.
var status = OutputCode(
currCode,
dst,
dstPos,
dstCap,
prevCode,
codeTab,
ref queue,
out firstByte,
out len
);
if (status != UnshrnkStatus.Ok)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
return status;
}
// Verify that the output matches walking the prefixes.
var c = currCode;
for (var i = 0; i < len; i++)
{
// assert(codeTab[c].len == len - i);
//assert(codeTab[c].extByte == dst[dstPos + len - i - 1]);
c = codeTab[c].prefixCode;
}
// Add a new code to the string table if there's room.
// The string is the previous code's string extended with the first byte of the current code's string.
newCode = CodeQueueRemoveNext(ref queue);
if (newCode != INVALID_CODE)
{
//assert(codeTab[prevCode].lastDstPos < dstPos);
codeTab[newCode].prefixCode = prevCode;
codeTab[newCode].extByte = firstByte;
codeTab[newCode].len = (ushort)(codeTab[prevCode].len + 1);
codeTab[newCode].lastDstPos = codeTab[prevCode].lastDstPos;
if (codeTab[prevCode].prefixCode == INVALID_CODE)
{
// prevCode was invalidated in a partial clearing. Until that code is re-used, the
// string represented by newCode is indeterminate.
codeTab[newCode].len = UNKNOWN_LEN;
}
// If prevCode was invalidated in a partial clearing, it's possible that newCode == prevCode,
// in which case it will never be used or cleared.
}
codeTab[currCode].lastDstPos = dstPos;
dstPos += len;
prevCode = currCode;
}
codeTab[currCode].lastDstPos = dstPos;
dstPos += len;
srcUsed = stream.BytesRead;
dstUsed = dstPos;
prevCode = currCode;
return UnshrnkStatus.Ok;
}
srcUsed = stream.BytesRead;
dstUsed = dstPos;
return UnshrnkStatus.Ok;
}
public enum UnshrnkStatus
{
Ok,
Full,
Error,
}
private struct CodeQueue
{
public int nextIdx;
public ushort[] codes;
}
private static void CodeQueueInit(ref CodeQueue q)
{
int codeQueueSize;
ushort code;
codeQueueSize = 0;
q.codes = new ushort[MAX_CODE - CONTROL_CODE + 2];
for (code = CONTROL_CODE + 1; code <= MAX_CODE; code++)
public enum UnshrnkStatus
{
q.codes[codeQueueSize++] = code;
Ok,
Full,
Error,
}
//assert(codeQueueSize < q.codes.Length);
q.codes[codeQueueSize] = INVALID_CODE; // End-of-queue marker.
q.nextIdx = 0;
}
private static ushort CodeQueueNext(ref CodeQueue q) =>
//assert(q.nextIdx < q.codes.Length);
q.codes[q.nextIdx];
private static ushort CodeQueueRemoveNext(ref CodeQueue q)
{
var code = CodeQueueNext(ref q);
if (code != INVALID_CODE)
private struct CodeQueue
{
q.nextIdx++;
public int nextIdx;
public ushort[] codes;
}
private static void CodeQueueInit(ref CodeQueue q)
{
int codeQueueSize;
ushort code;
codeQueueSize = 0;
q.codes = new ushort[MAX_CODE - CONTROL_CODE + 2];
for (code = CONTROL_CODE + 1; code <= MAX_CODE; code++)
{
q.codes[codeQueueSize++] = code;
}
//assert(codeQueueSize < q.codes.Length);
q.codes[codeQueueSize] = INVALID_CODE; // End-of-queue marker.
q.nextIdx = 0;
}
private static ushort CodeQueueNext(ref CodeQueue q) =>
//assert(q.nextIdx < q.codes.Length);
q.codes[q.nextIdx];
private static ushort CodeQueueRemoveNext(ref CodeQueue q)
{
var code = CodeQueueNext(ref q);
if (code != INVALID_CODE)
{
q.nextIdx++;
}
return code;
}
return code;
}
}

View File

@@ -7,138 +7,139 @@ using System.Threading.Tasks;
using SharpCompress.Compressors.RLE90;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Squeezed;
public class SqueezeStream : Stream, IStreamStack
namespace SharpCompress.Compressors.Squeezed
{
#if DEBUG_STREAMS
long IStreamStack.InstanceId { get; set; }
#endif
int IStreamStack.DefaultBufferSize { get; set; }
Stream IStreamStack.BaseStream() => _stream;
int IStreamStack.BufferSize
{
get => 0;
set { }
}
int IStreamStack.BufferPosition
{
get => 0;
set { }
}
void IStreamStack.SetPosition(long position) { }
private readonly Stream _stream;
private readonly int _compressedSize;
private const int NUMVALS = 257;
private const int SPEOF = 256;
private bool _processed = false;
public SqueezeStream(Stream stream, int compressedSize)
{
_stream = stream;
_compressedSize = compressedSize;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(SqueezeStream));
#endif
}
protected override void Dispose(bool disposing)
public class SqueezeStream : Stream, IStreamStack
{
#if DEBUG_STREAMS
this.DebugDispose(typeof(SqueezeStream));
long IStreamStack.InstanceId { get; set; }
#endif
base.Dispose(disposing);
}
int IStreamStack.DefaultBufferSize { get; set; }
public override bool CanRead => true;
Stream IStreamStack.BaseStream() => _stream;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotImplementedException();
public override long Position
{
get => _stream.Position;
set => throw new NotImplementedException();
}
public override void Flush() => throw new NotImplementedException();
public override int Read(byte[] buffer, int offset, int count)
{
if (_processed)
int IStreamStack.BufferSize
{
return 0;
get => 0;
set { }
}
_processed = true;
using var binaryReader = new BinaryReader(_stream);
// Read numnodes (equivalent to convert_u16!(numnodes, buf))
var numnodes = binaryReader.ReadUInt16();
// Validation: numnodes should be within bounds
if (numnodes >= NUMVALS)
int IStreamStack.BufferPosition
{
throw new InvalidDataException(
$"Invalid number of nodes {numnodes} (max {NUMVALS - 1})"
);
get => 0;
set { }
}
// Handle the case where no nodes exist
if (numnodes == 0)
void IStreamStack.SetPosition(long position) { }
private readonly Stream _stream;
private readonly int _compressedSize;
private const int NUMVALS = 257;
private const int SPEOF = 256;
private bool _processed = false;
public SqueezeStream(Stream stream, int compressedSize)
{
return 0;
_stream = stream;
_compressedSize = compressedSize;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(SqueezeStream));
#endif
}
// Build dnode (tree of nodes)
var dnode = new int[numnodes, 2];
for (int j = 0; j < numnodes; j++)
protected override void Dispose(bool disposing)
{
dnode[j, 0] = binaryReader.ReadInt16();
dnode[j, 1] = binaryReader.ReadInt16();
#if DEBUG_STREAMS
this.DebugDispose(typeof(SqueezeStream));
#endif
base.Dispose(disposing);
}
// Initialize BitReader for reading bits
var bitReader = new BitReader(_stream);
var decoded = new List<byte>();
public override bool CanRead => true;
int i = 0;
// Decode the buffer using the dnode tree
while (true)
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotImplementedException();
public override long Position
{
i = dnode[i, bitReader.ReadBit() ? 1 : 0];
if (i < 0)
get => _stream.Position;
set => throw new NotImplementedException();
}
public override void Flush() => throw new NotImplementedException();
public override int Read(byte[] buffer, int offset, int count)
{
if (_processed)
{
i = (short)-(i + 1);
if (i == SPEOF)
return 0;
}
_processed = true;
using var binaryReader = new BinaryReader(_stream);
// Read numnodes (equivalent to convert_u16!(numnodes, buf))
var numnodes = binaryReader.ReadUInt16();
// Validation: numnodes should be within bounds
if (numnodes >= NUMVALS)
{
throw new InvalidDataException(
$"Invalid number of nodes {numnodes} (max {NUMVALS - 1})"
);
}
// Handle the case where no nodes exist
if (numnodes == 0)
{
return 0;
}
// Build dnode (tree of nodes)
var dnode = new int[numnodes, 2];
for (int j = 0; j < numnodes; j++)
{
dnode[j, 0] = binaryReader.ReadInt16();
dnode[j, 1] = binaryReader.ReadInt16();
}
// Initialize BitReader for reading bits
var bitReader = new BitReader(_stream);
var decoded = new List<byte>();
int i = 0;
// Decode the buffer using the dnode tree
while (true)
{
i = dnode[i, bitReader.ReadBit() ? 1 : 0];
if (i < 0)
{
break;
}
else
{
decoded.Add((byte)i);
i = 0;
i = (short)-(i + 1);
if (i == SPEOF)
{
break;
}
else
{
decoded.Add((byte)i);
i = 0;
}
}
}
// Unpack the decoded buffer using the RLE class
var unpacked = RLE.UnpackRLE(decoded.ToArray());
unpacked.CopyTo(buffer, 0);
return unpacked.Count();
}
// Unpack the decoded buffer using the RLE class
var unpacked = RLE.UnpackRLE(decoded.ToArray());
unpacked.CopyTo(buffer, 0);
return unpacked.Count();
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();
public override void SetLength(long value) => throw new NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotImplementedException();
}
public override long Seek(long offset, SeekOrigin origin) =>
throw new NotImplementedException();
public override void SetLength(long value) => throw new NotImplementedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotImplementedException();
}

View File

@@ -22,9 +22,7 @@ public class XZFooter
public static XZFooter FromStream(Stream stream)
{
var footer = new XZFooter(
new BinaryReader(SharpCompressStream.Create(stream, leaveOpen: true), Encoding.UTF8)
);
var footer = new XZFooter(new BinaryReader(stream, Encoding.UTF8, true));
footer.Process();
return footer;
}

View File

@@ -18,9 +18,7 @@ public class XZHeader
public static XZHeader FromStream(Stream stream)
{
var header = new XZHeader(
new BinaryReader(SharpCompressStream.Create(stream, leaveOpen: true), Encoding.UTF8)
);
var header = new XZHeader(new BinaryReader(stream, Encoding.UTF8, true));
header.Process();
return header;
}

View File

@@ -32,7 +32,7 @@ public class XZIndex
public static XZIndex FromStream(Stream stream, bool indexMarkerAlreadyVerified)
{
var index = new XZIndex(
new BinaryReader(SharpCompressStream.Create(stream, leaveOpen: true), Encoding.UTF8),
new BinaryReader(stream, Encoding.UTF8, true),
indexMarkerAlreadyVerified
);
index.Process();

View File

@@ -1,311 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
// Some routines inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson:
// http://graphics.stanford.edu/~seander/bithacks.html
namespace System.Numerics
{
/// <summary>
/// Utility methods for intrinsic bit-twiddling operations.
/// The methods use hardware intrinsics when available on the underlying platform,
/// otherwise they use optimized software fallbacks.
/// </summary>
public static unsafe class BitOperations
{
// hack: should be public because of inline
public static readonly byte* TrailingZeroCountDeBruijn = GetArrayPointer(
new byte[]
{
00,
01,
28,
02,
29,
14,
24,
03,
30,
22,
20,
15,
25,
17,
04,
08,
31,
27,
13,
23,
21,
19,
16,
07,
26,
12,
18,
06,
11,
05,
10,
09,
}
);
// hack: should be public because of inline
public static readonly byte* Log2DeBruijn = GetArrayPointer(
new byte[]
{
00,
09,
01,
10,
13,
21,
02,
29,
11,
14,
16,
18,
22,
25,
03,
30,
08,
12,
20,
28,
15,
17,
24,
07,
19,
27,
23,
06,
26,
05,
04,
31,
}
);
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(uint value)
{
// The 0->0 contract is fulfilled by setting the LSB to 1.
// Log(1) is 0, and setting the LSB for values > 1 does not change the log2 result.
value |= 1;
// value lzcnt actual expected
// ..0001 31 31-31 0
// ..0010 30 31-30 1
// 0010.. 2 31-2 29
// 0100.. 1 31-1 30
// 1000.. 0 31-0 31
// Fallback contract is 0->0
// No AggressiveInlining due to large method size
// Has conventional contract 0->0 (Log(0) is undefined)
// Fill trailing zeros with ones, eg 00010010 becomes 00011111
value |= value >> 01;
value |= value >> 02;
value |= value >> 04;
value |= value >> 08;
value |= value >> 16;
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return Log2DeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_1100_0100_1010_1100_1101_1101u
(int)((value * 0x07C4ACDDu) >> 27)
];
}
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(ulong value)
{
value |= 1;
uint hi = (uint)(value >> 32);
if (hi == 0)
{
return Log2((uint)value);
}
return 32 + Log2(hi);
}
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(int value) => TrailingZeroCount((uint)value);
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(uint value)
{
// Unguarded fallback contract is 0->0, BSF contract is 0->undefined
if (value == 0)
{
return 32;
}
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return TrailingZeroCountDeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_0111_1100_1011_0101_0011_0001u
(int)(((value & (uint)-(int)value) * 0x077CB531u) >> 27)
]; // Multi-cast mitigates redundant conv.u8
}
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(long value) => TrailingZeroCount((ulong)value);
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(ulong value)
{
uint lo = (uint)value;
if (lo == 0)
{
return 32 + TrailingZeroCount((uint)(value >> 32));
}
return TrailingZeroCount(lo);
}
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateLeft(uint value, int offset) =>
(value << offset) | (value >> (32 - offset));
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateLeft(ulong value, int offset) =>
(value << offset) | (value >> (64 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateRight(uint value, int offset) =>
(value >> offset) | (value << (32 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateRight(ulong value, int offset) =>
(value >> offset) | (value << (64 - offset));
/// <summary>
/// Count the number of leading zero bits in a mask.
/// Similar in behavior to the x86 instruction LZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LeadingZeroCount(uint value)
{
// Unguarded fallback contract is 0->31, BSR contract is 0->undefined
if (value == 0)
{
return 32;
}
// No AggressiveInlining due to large method size
// Has conventional contract 0->0 (Log(0) is undefined)
// Fill trailing zeros with ones, eg 00010010 becomes 00011111
value |= value >> 01;
value |= value >> 02;
value |= value >> 04;
value |= value >> 08;
value |= value >> 16;
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return 31
^ Log2DeBruijn[
// uint|long -> IntPtr cast on 32-bit platforms does expensive overflow checks not needed here
(int)((value * 0x07C4ACDDu) >> 27)
];
}
/// <summary>
/// Count the number of leading zero bits in a mask.
/// Similar in behavior to the x86 instruction LZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int LeadingZeroCount(ulong value)
{
uint hi = (uint)(value >> 32);
if (hi == 0)
{
return 32 + LeadingZeroCount((uint)value);
}
return LeadingZeroCount(hi);
}
}
}
#endif

View File

@@ -1,301 +0,0 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
public class CompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] outputBuffer;
private readonly bool preserveCompressor;
private readonly bool leaveOpen;
private Compressor? compressor;
private ZSTD_outBuffer_s output;
public CompressionStream(
Stream stream,
int level = Compressor.DefaultCompressionLevel,
int bufferSize = 0,
bool leaveOpen = true
)
: this(stream, new Compressor(level), bufferSize, false, leaveOpen) { }
public CompressionStream(
Stream stream,
Compressor compressor,
int bufferSize = 0,
bool preserveCompressor = true,
bool leaveOpen = true
)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanWrite)
throw new ArgumentException("Stream is not writable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
this.compressor = compressor;
this.preserveCompressor = preserveCompressor;
this.leaveOpen = leaveOpen;
var outputBufferSize =
bufferSize > 0
? bufferSize
: (int)Unsafe.Methods.ZSTD_CStreamOutSize().EnsureZstdSuccess();
outputBuffer = ArrayPool<byte>.Shared.Rent(outputBufferSize);
output = new ZSTD_outBuffer_s { pos = 0, size = (nuint)outputBufferSize };
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
EnsureNotDisposed();
compressor.NotNull().SetParameter(parameter, value);
}
public int GetParameter(ZSTD_cParameter parameter)
{
EnsureNotDisposed();
return compressor.NotNull().GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
compressor.NotNull().LoadDictionary(dict);
}
~CompressionStream() => Dispose(false);
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override async ValueTask DisposeAsync()
#else
public async Task DisposeAsync()
#endif
{
if (compressor == null)
return;
try
{
await FlushInternalAsync(ZSTD_EndDirective.ZSTD_e_end).ConfigureAwait(false);
}
finally
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
}
protected override void Dispose(bool disposing)
{
if (compressor == null)
return;
try
{
if (disposing)
FlushInternal(ZSTD_EndDirective.ZSTD_e_end);
}
finally
{
ReleaseUnmanagedResources();
}
}
private void ReleaseUnmanagedResources()
{
if (!preserveCompressor)
{
compressor.NotNull().Dispose();
}
compressor = null;
if (outputBuffer != null)
{
ArrayPool<byte>.Shared.Return(outputBuffer);
}
if (!leaveOpen)
{
innerStream.Dispose();
}
}
public override void Flush() => FlushInternal(ZSTD_EndDirective.ZSTD_e_flush);
public override async Task FlushAsync(CancellationToken cancellationToken) =>
await FlushInternalAsync(ZSTD_EndDirective.ZSTD_e_flush, cancellationToken)
.ConfigureAwait(false);
private void FlushInternal(ZSTD_EndDirective directive) => WriteInternal(null, directive);
private async Task FlushInternalAsync(
ZSTD_EndDirective directive,
CancellationToken cancellationToken = default
) => await WriteInternalAsync(null, directive, cancellationToken).ConfigureAwait(false);
public override void Write(byte[] buffer, int offset, int count) =>
Write(new ReadOnlySpan<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override void Write(ReadOnlySpan<byte> buffer) =>
WriteInternal(buffer, ZSTD_EndDirective.ZSTD_e_continue);
#else
public void Write(ReadOnlySpan<byte> buffer) =>
WriteInternal(buffer, ZSTD_EndDirective.ZSTD_e_continue);
#endif
private void WriteInternal(ReadOnlySpan<byte> buffer, ZSTD_EndDirective directive)
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s
{
pos = 0,
size = buffer != null ? (nuint)buffer.Length : 0,
};
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(ref input, buffer, directive);
var written = (int)output.pos;
if (written > 0)
innerStream.Write(outputBuffer, 0, written);
} while (
directive == ZSTD_EndDirective.ZSTD_e_continue ? input.pos < input.size : remaining > 0
);
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
private async ValueTask WriteInternalAsync(
ReadOnlyMemory<byte>? buffer,
ZSTD_EndDirective directive,
CancellationToken cancellationToken = default
)
#else
private async Task WriteInternalAsync(
ReadOnlyMemory<byte>? buffer,
ZSTD_EndDirective directive,
CancellationToken cancellationToken = default
)
#endif
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s
{
pos = 0,
size = buffer.HasValue ? (nuint)buffer.Value.Length : 0,
};
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(
ref input,
buffer.HasValue ? buffer.Value.Span : null,
directive
);
var written = (int)output.pos;
if (written > 0)
await innerStream
.WriteAsync(outputBuffer, 0, written, cancellationToken)
.ConfigureAwait(false);
} while (
directive == ZSTD_EndDirective.ZSTD_e_continue ? input.pos < input.size : remaining > 0
);
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override Task WriteAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
) => WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken).AsTask();
public override async ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default
) =>
await WriteInternalAsync(buffer, ZSTD_EndDirective.ZSTD_e_continue, cancellationToken)
.ConfigureAwait(false);
#else
public override Task WriteAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
) => WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken);
public async Task WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default
) =>
await WriteInternalAsync(buffer, ZSTD_EndDirective.ZSTD_e_continue, cancellationToken)
.ConfigureAwait(false);
#endif
internal unsafe nuint CompressStream(
ref ZSTD_inBuffer_s input,
ReadOnlySpan<byte> inputBuffer,
ZSTD_EndDirective directive
)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return compressor
.NotNull()
.CompressStream(ref input, ref output, directive)
.EnsureZstdSuccess();
}
}
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override int Read(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (compressor == null)
throw new ObjectDisposedException(nameof(CompressionStream));
}
public void SetPledgedSrcSize(ulong pledgedSrcSize)
{
EnsureNotDisposed();
compressor.NotNull().SetPledgedSrcSize(pledgedSrcSize);
}
}

View File

@@ -1,204 +0,0 @@
using System;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
public unsafe class Compressor : IDisposable
{
/// <summary>
/// Minimum negative compression level allowed
/// </summary>
public static int MinCompressionLevel => Unsafe.Methods.ZSTD_minCLevel();
/// <summary>
/// Maximum compression level available
/// </summary>
public static int MaxCompressionLevel => Unsafe.Methods.ZSTD_maxCLevel();
/// <summary>
/// Default compression level
/// </summary>
/// <see cref="Unsafe.Methods.ZSTD_defaultCLevel"/>
public const int DefaultCompressionLevel = 3;
private int level = DefaultCompressionLevel;
private readonly SafeCctxHandle handle;
public int Level
{
get => level;
set
{
if (level != value)
{
level = value;
SetParameter(ZSTD_cParameter.ZSTD_c_compressionLevel, value);
}
}
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
using var cctx = handle.Acquire();
Unsafe.Methods.ZSTD_CCtx_setParameter(cctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_cParameter parameter)
{
using var cctx = handle.Acquire();
int value;
Unsafe.Methods.ZSTD_CCtx_getParameter(cctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
var dictReadOnlySpan = new ReadOnlySpan<byte>(dict);
LoadDictionary(dictReadOnlySpan);
}
public void LoadDictionary(ReadOnlySpan<byte> dict)
{
using var cctx = handle.Acquire();
fixed (byte* dictPtr = dict)
Unsafe
.Methods.ZSTD_CCtx_loadDictionary(cctx, dictPtr, (nuint)dict.Length)
.EnsureZstdSuccess();
}
public Compressor(int level = DefaultCompressionLevel)
{
handle = SafeCctxHandle.Create();
Level = level;
}
public static int GetCompressBound(int length) =>
(int)Unsafe.Methods.ZSTD_compressBound((nuint)length);
public static ulong GetCompressBoundLong(ulong length) =>
Unsafe.Methods.ZSTD_compressBound((nuint)length);
public Span<byte> Wrap(ReadOnlySpan<byte> src)
{
var dest = new byte[GetCompressBound(src.Length)];
var length = Wrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Wrap(byte[] src, byte[] dest, int offset) =>
Wrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Wrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
using var cctx = handle.Acquire();
return (int)
Unsafe
.Methods.ZSTD_compress2(
cctx,
destPtr,
(nuint)dest.Length,
srcPtr,
(nuint)src.Length
)
.EnsureZstdSuccess();
}
}
public int Wrap(ArraySegment<byte> src, ArraySegment<byte> dest) =>
Wrap((ReadOnlySpan<byte>)src, dest);
public int Wrap(
byte[] src,
int srcOffset,
int srcLength,
byte[] dst,
int dstOffset,
int dstLength
) =>
Wrap(
new ReadOnlySpan<byte>(src, srcOffset, srcLength),
new Span<byte>(dst, dstOffset, dstLength)
);
public bool TryWrap(byte[] src, byte[] dest, int offset, out int written) =>
TryWrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryWrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
nuint returnValue;
using (var cctx = handle.Acquire())
{
returnValue = Unsafe.Methods.ZSTD_compress2(
cctx,
destPtr,
(nuint)dest.Length,
srcPtr,
(nuint)src.Length
);
}
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int)returnValue;
return true;
}
}
public bool TryWrap(ArraySegment<byte> src, ArraySegment<byte> dest, out int written) =>
TryWrap((ReadOnlySpan<byte>)src, dest, out written);
public bool TryWrap(
byte[] src,
int srcOffset,
int srcLength,
byte[] dst,
int dstOffset,
int dstLength,
out int written
) =>
TryWrap(
new ReadOnlySpan<byte>(src, srcOffset, srcLength),
new Span<byte>(dst, dstOffset, dstLength),
out written
);
public void Dispose()
{
handle.Dispose();
GC.SuppressFinalize(this);
}
internal nuint CompressStream(
ref ZSTD_inBuffer_s input,
ref ZSTD_outBuffer_s output,
ZSTD_EndDirective directive
)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
using var cctx = handle.Acquire();
return Unsafe
.Methods.ZSTD_compressStream2(cctx, outputPtr, inputPtr, directive)
.EnsureZstdSuccess();
}
}
public void SetPledgedSrcSize(ulong pledgedSrcSize)
{
using var cctx = handle.Acquire();
Unsafe.Methods.ZSTD_CCtx_setPledgedSrcSize(cctx, pledgedSrcSize).EnsureZstdSuccess();
}
}

View File

@@ -1,8 +0,0 @@
namespace SharpCompress.Compressors.ZStandard;
internal class Constants
{
//NOTE: https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element#remarks
//NOTE: https://github.com/dotnet/runtime/blob/v5.0.0-rtm.20519.4/src/libraries/System.Private.CoreLib/src/System/Array.cs#L27
public const ulong MaxByteArrayLength = 0x7FFFFFC7;
}

View File

@@ -1,293 +0,0 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
public class DecompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] inputBuffer;
private readonly int inputBufferSize;
private readonly bool preserveDecompressor;
private readonly bool leaveOpen;
private readonly bool checkEndOfStream;
private Decompressor? decompressor;
private ZSTD_inBuffer_s input;
private nuint lastDecompressResult = 0;
private bool contextDrained = true;
public DecompressionStream(
Stream stream,
int bufferSize = 0,
bool checkEndOfStream = true,
bool leaveOpen = true
)
: this(stream, new Decompressor(), bufferSize, checkEndOfStream, false, leaveOpen) { }
public DecompressionStream(
Stream stream,
Decompressor decompressor,
int bufferSize = 0,
bool checkEndOfStream = true,
bool preserveDecompressor = true,
bool leaveOpen = true
)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanRead)
throw new ArgumentException("Stream is not readable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
this.decompressor = decompressor;
this.preserveDecompressor = preserveDecompressor;
this.leaveOpen = leaveOpen;
this.checkEndOfStream = checkEndOfStream;
inputBufferSize =
bufferSize > 0
? bufferSize
: (int)Unsafe.Methods.ZSTD_DStreamInSize().EnsureZstdSuccess();
inputBuffer = ArrayPool<byte>.Shared.Rent(inputBufferSize);
input = new ZSTD_inBuffer_s { pos = (nuint)inputBufferSize, size = (nuint)inputBufferSize };
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
EnsureNotDisposed();
decompressor.NotNull().SetParameter(parameter, value);
}
public int GetParameter(ZSTD_dParameter parameter)
{
EnsureNotDisposed();
return decompressor.NotNull().GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
decompressor.NotNull().LoadDictionary(dict);
}
~DecompressionStream() => Dispose(false);
protected override void Dispose(bool disposing)
{
if (decompressor == null)
return;
if (!preserveDecompressor)
{
decompressor.Dispose();
}
decompressor = null;
if (inputBuffer != null)
{
ArrayPool<byte>.Shared.Return(inputBuffer);
}
if (!leaveOpen)
{
innerStream.Dispose();
}
}
public override int Read(byte[] buffer, int offset, int count) =>
Read(new Span<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override int Read(Span<byte> buffer)
#else
public int Read(Span<byte> buffer)
#endif
{
EnsureNotDisposed();
// Guard against infinite loop (output.pos would never become non-zero)
if (buffer.Length == 0)
{
return 0;
}
var output = new ZSTD_outBuffer_s { pos = 0, size = (nuint)buffer.Length };
while (true)
{
// If there is still input available, or there might be data buffered in the decompressor context, flush that out
while (input.pos < input.size || !contextDrained)
{
nuint oldInputPos = input.pos;
nuint result = DecompressStream(ref output, buffer);
if (output.pos > 0 || oldInputPos != input.pos)
{
// Keep result from last decompress call that made some progress, so we known if we're at end of frame
lastDecompressResult = result;
}
// If decompression filled the output buffer, there might still be data buffered in the decompressor context
contextDrained = output.pos < output.size;
// If we have data to return, return it immediately, so we won't stall on Read
if (output.pos > 0)
{
return (int)output.pos;
}
}
// Otherwise, read some more input
int bytesRead;
if ((bytesRead = innerStream.Read(inputBuffer, 0, inputBufferSize)) == 0)
{
if (checkEndOfStream && lastDecompressResult != 0)
{
throw new EndOfStreamException("Premature end of stream");
}
return 0;
}
input.size = (nuint)bytesRead;
input.pos = 0;
}
}
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
) => ReadAsync(new Memory<byte>(buffer, offset, count), cancellationToken).AsTask();
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
#else
public override Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
) => ReadAsync(new Memory<byte>(buffer, offset, count), cancellationToken);
public async Task<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
#endif
{
EnsureNotDisposed();
// Guard against infinite loop (output.pos would never become non-zero)
if (buffer.Length == 0)
{
return 0;
}
var output = new ZSTD_outBuffer_s { pos = 0, size = (nuint)buffer.Length };
while (true)
{
// If there is still input available, or there might be data buffered in the decompressor context, flush that out
while (input.pos < input.size || !contextDrained)
{
nuint oldInputPos = input.pos;
nuint result = DecompressStream(ref output, buffer.Span);
if (output.pos > 0 || oldInputPos != input.pos)
{
// Keep result from last decompress call that made some progress, so we known if we're at end of frame
lastDecompressResult = result;
}
// If decompression filled the output buffer, there might still be data buffered in the decompressor context
contextDrained = output.pos < output.size;
// If we have data to return, return it immediately, so we won't stall on Read
if (output.pos > 0)
{
return (int)output.pos;
}
}
// Otherwise, read some more input
int bytesRead;
if (
(
bytesRead = await innerStream
.ReadAsync(inputBuffer, 0, inputBufferSize, cancellationToken)
.ConfigureAwait(false)
) == 0
)
{
if (checkEndOfStream && lastDecompressResult != 0)
{
throw new EndOfStreamException("Premature end of stream");
}
return 0;
}
input.size = (nuint)bytesRead;
input.pos = 0;
}
}
private unsafe nuint DecompressStream(ref ZSTD_outBuffer_s output, Span<byte> outputBuffer)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return decompressor.NotNull().DecompressStream(ref input, ref output);
}
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Flush() => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) =>
throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (decompressor == null)
throw new ObjectDisposedException(nameof(DecompressionStream));
}
#if NETSTANDARD2_0 || NETFRAMEWORK
public virtual Task DisposeAsync()
{
try
{
Dispose();
return Task.CompletedTask;
}
catch (Exception exc)
{
return Task.FromException(exc);
}
}
#endif
}

View File

@@ -1,176 +0,0 @@
using System;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
public unsafe class Decompressor : IDisposable
{
private readonly SafeDctxHandle handle;
public Decompressor()
{
handle = SafeDctxHandle.Create();
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
using var dctx = handle.Acquire();
Unsafe.Methods.ZSTD_DCtx_setParameter(dctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_dParameter parameter)
{
using var dctx = handle.Acquire();
int value;
Unsafe.Methods.ZSTD_DCtx_getParameter(dctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
var dictReadOnlySpan = new ReadOnlySpan<byte>(dict);
this.LoadDictionary(dictReadOnlySpan);
}
public void LoadDictionary(ReadOnlySpan<byte> dict)
{
using var dctx = handle.Acquire();
fixed (byte* dictPtr = dict)
Unsafe
.Methods.ZSTD_DCtx_loadDictionary(dctx, dictPtr, (nuint)dict.Length)
.EnsureZstdSuccess();
}
public static ulong GetDecompressedSize(ReadOnlySpan<byte> src)
{
fixed (byte* srcPtr = src)
return Unsafe
.Methods.ZSTD_decompressBound(srcPtr, (nuint)src.Length)
.EnsureContentSizeOk();
}
public static ulong GetDecompressedSize(ArraySegment<byte> src) =>
GetDecompressedSize((ReadOnlySpan<byte>)src);
public static ulong GetDecompressedSize(byte[] src, int srcOffset, int srcLength) =>
GetDecompressedSize(new ReadOnlySpan<byte>(src, srcOffset, srcLength));
public Span<byte> Unwrap(ReadOnlySpan<byte> src, int maxDecompressedSize = int.MaxValue)
{
var expectedDstSize = GetDecompressedSize(src);
if (expectedDstSize > (ulong)maxDecompressedSize)
throw new ZstdException(
ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than {nameof(maxDecompressedSize)} {maxDecompressedSize}"
);
if (expectedDstSize > Constants.MaxByteArrayLength)
throw new ZstdException(
ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than max possible byte array size {Constants.MaxByteArrayLength}"
);
var dest = new byte[expectedDstSize];
var length = Unwrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Unwrap(byte[] src, byte[] dest, int offset) =>
Unwrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Unwrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
using var dctx = handle.Acquire();
return (int)
Unsafe
.Methods.ZSTD_decompressDCtx(
dctx,
destPtr,
(nuint)dest.Length,
srcPtr,
(nuint)src.Length
)
.EnsureZstdSuccess();
}
}
public int Unwrap(
byte[] src,
int srcOffset,
int srcLength,
byte[] dst,
int dstOffset,
int dstLength
) =>
Unwrap(
new ReadOnlySpan<byte>(src, srcOffset, srcLength),
new Span<byte>(dst, dstOffset, dstLength)
);
public bool TryUnwrap(byte[] src, byte[] dest, int offset, out int written) =>
TryUnwrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryUnwrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
nuint returnValue;
using (var dctx = handle.Acquire())
{
returnValue = Unsafe.Methods.ZSTD_decompressDCtx(
dctx,
destPtr,
(nuint)dest.Length,
srcPtr,
(nuint)src.Length
);
}
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int)returnValue;
return true;
}
}
public bool TryUnwrap(
byte[] src,
int srcOffset,
int srcLength,
byte[] dst,
int dstOffset,
int dstLength,
out int written
) =>
TryUnwrap(
new ReadOnlySpan<byte>(src, srcOffset, srcLength),
new Span<byte>(dst, dstOffset, dstLength),
out written
);
public void Dispose()
{
handle.Dispose();
GC.SuppressFinalize(this);
}
internal nuint DecompressStream(ref ZSTD_inBuffer_s input, ref ZSTD_outBuffer_s output)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
using var dctx = handle.Acquire();
return Unsafe
.Methods.ZSTD_decompressStream(dctx, outputPtr, inputPtr)
.EnsureZstdSuccess();
}
}
}

View File

@@ -1,141 +0,0 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
namespace SharpCompress.Compressors.ZStandard;
internal unsafe class JobThreadPool : IDisposable
{
private int numThreads;
private readonly List<JobThread> threads;
private readonly BlockingCollection<Job> queue;
private struct Job
{
public void* function;
public void* opaque;
}
private class JobThread
{
private Thread Thread { get; }
public CancellationTokenSource CancellationTokenSource { get; }
public JobThread(Thread thread)
{
CancellationTokenSource = new CancellationTokenSource();
Thread = thread;
}
public void Start()
{
Thread.Start(this);
}
public void Cancel()
{
CancellationTokenSource.Cancel();
}
public void Join()
{
Thread.Join();
}
}
private void Worker(object? obj)
{
if (obj is not JobThread poolThread)
return;
var cancellationToken = poolThread.CancellationTokenSource.Token;
while (!queue.IsCompleted && !cancellationToken.IsCancellationRequested)
{
try
{
if (queue.TryTake(out var job, -1, cancellationToken))
((delegate* managed<void*, void>)job.function)(job.opaque);
}
catch (InvalidOperationException) { }
catch (OperationCanceledException) { }
}
}
public JobThreadPool(int num, int queueSize)
{
numThreads = num;
queue = new BlockingCollection<Job>(queueSize + 1);
threads = new List<JobThread>(num);
for (var i = 0; i < numThreads; i++)
CreateThread();
}
private void CreateThread()
{
var poolThread = new JobThread(new Thread(Worker));
threads.Add(poolThread);
poolThread.Start();
}
public void Resize(int num)
{
lock (threads)
{
if (num < numThreads)
{
for (var i = numThreads - 1; i >= num; i--)
{
threads[i].Cancel();
threads.RemoveAt(i);
}
}
else
{
for (var i = numThreads; i < num; i++)
CreateThread();
}
}
numThreads = num;
}
public void Add(void* function, void* opaque)
{
queue.Add(new Job { function = function, opaque = opaque });
}
public bool TryAdd(void* function, void* opaque)
{
return queue.TryAdd(new Job { function = function, opaque = opaque });
}
public void Join(bool cancel = true)
{
queue.CompleteAdding();
List<JobThread> jobThreads;
lock (threads)
jobThreads = new List<JobThread>(threads);
if (cancel)
{
foreach (var thread in jobThreads)
thread.Cancel();
}
foreach (var thread in jobThreads)
thread.Join();
}
public void Dispose()
{
queue.Dispose();
}
public int Size()
{
// todo not implemented
// https://github.com/dotnet/runtime/issues/24200
return 0;
}
}

View File

@@ -1,163 +0,0 @@
using System;
using System.Runtime.InteropServices;
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
/// <summary>
/// Provides the base class for ZstdSharp <see cref="SafeHandle"/> implementations.
/// </summary>
/// <remarks>
/// Even though ZstdSharp is a managed library, its internals are using unmanaged
/// memory and we are using safe handles in the library's high-level API to ensure
/// proper disposal of unmanaged resources and increase safety.
/// </remarks>
/// <seealso cref="SafeCctxHandle"/>
/// <seealso cref="SafeDctxHandle"/>
internal abstract unsafe class SafeZstdHandle : SafeHandle
{
/// <summary>
/// Parameterless constructor is hidden. Use the static <c>Create</c> factory
/// method to create a new safe handle instance.
/// </summary>
protected SafeZstdHandle()
: base(IntPtr.Zero, true) { }
public sealed override bool IsInvalid => handle == IntPtr.Zero;
}
/// <summary>
/// Safely wraps an unmanaged Zstd compression context.
/// </summary>
internal sealed unsafe class SafeCctxHandle : SafeZstdHandle
{
/// <inheritdoc/>
private SafeCctxHandle() { }
/// <summary>
/// Creates a new instance of <see cref="SafeCctxHandle"/>.
/// </summary>
/// <returns></returns>
/// <exception cref="ZstdException">Creation failed.</exception>
public static SafeCctxHandle Create()
{
var safeHandle = new SafeCctxHandle();
bool success = false;
try
{
var cctx = Unsafe.Methods.ZSTD_createCCtx();
if (cctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create cctx");
safeHandle.SetHandle((IntPtr)cctx);
success = true;
}
finally
{
if (!success)
{
safeHandle.SetHandleAsInvalid();
}
}
return safeHandle;
}
/// <summary>
/// Acquires a reference to the safe handle.
/// </summary>
/// <returns>
/// A <see cref="SafeHandleHolder{T}"/> instance that can be implicitly converted to a pointer
/// to <see cref="ZSTD_CCtx_s"/>.
/// </returns>
public SafeHandleHolder<ZSTD_CCtx_s> Acquire() => new(this);
protected override bool ReleaseHandle()
{
return Unsafe.Methods.ZSTD_freeCCtx((ZSTD_CCtx_s*)handle) == 0;
}
}
/// <summary>
/// Safely wraps an unmanaged Zstd compression context.
/// </summary>
internal sealed unsafe class SafeDctxHandle : SafeZstdHandle
{
/// <inheritdoc/>
private SafeDctxHandle() { }
/// <summary>
/// Creates a new instance of <see cref="SafeDctxHandle"/>.
/// </summary>
/// <returns></returns>
/// <exception cref="ZstdException">Creation failed.</exception>
public static SafeDctxHandle Create()
{
var safeHandle = new SafeDctxHandle();
bool success = false;
try
{
var dctx = Unsafe.Methods.ZSTD_createDCtx();
if (dctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create dctx");
safeHandle.SetHandle((IntPtr)dctx);
success = true;
}
finally
{
if (!success)
{
safeHandle.SetHandleAsInvalid();
}
}
return safeHandle;
}
/// <summary>
/// Acquires a reference to the safe handle.
/// </summary>
/// <returns>
/// A <see cref="SafeHandleHolder{T}"/> instance that can be implicitly converted to a pointer
/// to <see cref="ZSTD_DCtx_s"/>.
/// </returns>
public SafeHandleHolder<ZSTD_DCtx_s> Acquire() => new(this);
protected override bool ReleaseHandle()
{
return Unsafe.Methods.ZSTD_freeDCtx((ZSTD_DCtx_s*)handle) == 0;
}
}
/// <summary>
/// Provides a convenient interface to safely acquire pointers of a specific type
/// from a <see cref="SafeHandle"/>, by utilizing <see langword="using"/> blocks.
/// </summary>
/// <typeparam name="T">The type of pointers to return.</typeparam>
/// <remarks>
/// Safe handle holders can be <see cref="Dispose"/>d to decrement the safe handle's
/// reference count, and can be implicitly converted to pointers to <see cref="T"/>.
/// </remarks>
internal unsafe ref struct SafeHandleHolder<T>
where T : unmanaged
{
private readonly SafeHandle _handle;
private bool _refAdded;
public SafeHandleHolder(SafeHandle safeHandle)
{
_handle = safeHandle;
_refAdded = false;
safeHandle.DangerousAddRef(ref _refAdded);
}
public static implicit operator T*(SafeHandleHolder<T> holder) =>
(T*)holder._handle.DangerousGetHandle();
public void Dispose()
{
if (_refAdded)
{
_handle.DangerousRelease();
_refAdded = false;
}
}
}

View File

@@ -1,22 +0,0 @@
using System.Threading;
namespace SharpCompress.Compressors.ZStandard;
internal static unsafe class SynchronizationWrapper
{
private static object UnwrapObject(void** obj) => UnmanagedObject.Unwrap<object>(*obj);
public static void Init(void** obj) => *obj = UnmanagedObject.Wrap(new object());
public static void Free(void** obj) => UnmanagedObject.Free(*obj);
public static void Enter(void** obj) => Monitor.Enter(UnwrapObject(obj));
public static void Exit(void** obj) => Monitor.Exit(UnwrapObject(obj));
public static void Pulse(void** obj) => Monitor.Pulse(UnwrapObject(obj));
public static void PulseAll(void** obj) => Monitor.PulseAll(UnwrapObject(obj));
public static void Wait(void** mutex) => Monitor.Wait(UnwrapObject(mutex));
}

View File

@@ -1,48 +0,0 @@
using SharpCompress.Compressors.ZStandard.Unsafe;
namespace SharpCompress.Compressors.ZStandard;
public static unsafe class ThrowHelper
{
private const ulong ZSTD_CONTENTSIZE_UNKNOWN = unchecked(0UL - 1);
private const ulong ZSTD_CONTENTSIZE_ERROR = unchecked(0UL - 2);
public static nuint EnsureZstdSuccess(this nuint returnValue)
{
if (Unsafe.Methods.ZSTD_isError(returnValue))
ThrowException(returnValue, Unsafe.Methods.ZSTD_getErrorName(returnValue));
return returnValue;
}
public static nuint EnsureZdictSuccess(this nuint returnValue)
{
if (Unsafe.Methods.ZDICT_isError(returnValue))
ThrowException(returnValue, Unsafe.Methods.ZDICT_getErrorName(returnValue));
return returnValue;
}
public static ulong EnsureContentSizeOk(this ulong returnValue)
{
if (returnValue == ZSTD_CONTENTSIZE_UNKNOWN)
throw new ZstdException(
ZSTD_ErrorCode.ZSTD_error_GENERIC,
"Decompressed content size is not specified"
);
if (returnValue == ZSTD_CONTENTSIZE_ERROR)
throw new ZstdException(
ZSTD_ErrorCode.ZSTD_error_GENERIC,
"Decompressed content size cannot be determined (e.g. invalid magic number, srcSize too small)"
);
return returnValue;
}
private static void ThrowException(nuint returnValue, string message)
{
var code = 0 - returnValue;
throw new ZstdException((ZSTD_ErrorCode)code, message);
}
}

View File

@@ -1,18 +0,0 @@
using System;
using System.Runtime.InteropServices;
namespace SharpCompress.Compressors.ZStandard;
/*
* Wrap object to void* to make it unmanaged
*/
internal static unsafe class UnmanagedObject
{
public static void* Wrap(object obj) => (void*)GCHandle.ToIntPtr(GCHandle.Alloc(obj));
private static GCHandle UnwrapGcHandle(void* value) => GCHandle.FromIntPtr((IntPtr)value);
public static T Unwrap<T>(void* value) => (T)UnwrapGcHandle(value).Target!;
public static void Free(void* value) => UnwrapGcHandle(value).Free();
}

View File

@@ -1,52 +0,0 @@
using System.Runtime.CompilerServices;
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
/* custom memory allocation functions */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void* ZSTD_customMalloc(nuint size, ZSTD_customMem customMem)
{
if (customMem.customAlloc != null)
return ((delegate* managed<void*, nuint, void*>)customMem.customAlloc)(
customMem.opaque,
size
);
return malloc(size);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void* ZSTD_customCalloc(nuint size, ZSTD_customMem customMem)
{
if (customMem.customAlloc != null)
{
/* calloc implemented as malloc+memset;
* not as efficient as calloc, but next best guess for custom malloc */
void* ptr = ((delegate* managed<void*, nuint, void*>)customMem.customAlloc)(
customMem.opaque,
size
);
memset(ptr, 0, (uint)size);
return ptr;
}
return calloc(1, size);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
{
if (ptr != null)
{
if (customMem.customFree != null)
((delegate* managed<void*, void*, void>)customMem.customFree)(
customMem.opaque,
ptr
);
else
free(ptr);
}
}
}

View File

@@ -1,14 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
public unsafe struct BIT_CStream_t
{
public nuint bitContainer;
public uint bitPos;
public sbyte* startPtr;
public sbyte* ptr;
public sbyte* endPtr;
}

View File

@@ -1,16 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public enum BIT_DStream_status
{
/* fully refilled */
BIT_DStream_unfinished = 0,
/* still some bits left in bitstream */
BIT_DStream_endOfBuffer = 1,
/* bitstream entirely consumed, bit-exact */
BIT_DStream_completed = 2,
/* user requested more bits than present in bitstream */
BIT_DStream_overflow = 3,
}

View File

@@ -1,13 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
public unsafe struct BIT_DStream_t
{
public nuint bitContainer;
public uint bitsConsumed;
public sbyte* ptr;
public sbyte* start;
public sbyte* limitPtr;
}

View File

@@ -1,60 +0,0 @@
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_countTrailingZeros32(uint val)
{
assert(val != 0);
return (uint)BitOperations.TrailingZeroCount(val);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_countLeadingZeros32(uint val)
{
assert(val != 0);
return (uint)BitOperations.LeadingZeroCount(val);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_countTrailingZeros64(ulong val)
{
assert(val != 0);
return (uint)BitOperations.TrailingZeroCount(val);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_countLeadingZeros64(ulong val)
{
assert(val != 0);
return (uint)BitOperations.LeadingZeroCount(val);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_NbCommonBytes(nuint val)
{
assert(val != 0);
if (BitConverter.IsLittleEndian)
{
return MEM_64bits
? (uint)BitOperations.TrailingZeroCount(val) >> 3
: (uint)BitOperations.TrailingZeroCount((uint)val) >> 3;
}
return MEM_64bits
? (uint)BitOperations.LeadingZeroCount(val) >> 3
: (uint)BitOperations.LeadingZeroCount((uint)val) >> 3;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ZSTD_highbit32(uint val)
{
assert(val != 0);
return (uint)BitOperations.Log2(val);
}
}

View File

@@ -1,739 +0,0 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
#if NETCOREAPP3_0_OR_GREATER
using System.Runtime.Intrinsics.X86;
#endif
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
#if NET7_0_OR_GREATER
private static ReadOnlySpan<uint> Span_BIT_mask =>
new uint[32]
{
0,
1,
3,
7,
0xF,
0x1F,
0x3F,
0x7F,
0xFF,
0x1FF,
0x3FF,
0x7FF,
0xFFF,
0x1FFF,
0x3FFF,
0x7FFF,
0xFFFF,
0x1FFFF,
0x3FFFF,
0x7FFFF,
0xFFFFF,
0x1FFFFF,
0x3FFFFF,
0x7FFFFF,
0xFFFFFF,
0x1FFFFFF,
0x3FFFFFF,
0x7FFFFFF,
0xFFFFFFF,
0x1FFFFFFF,
0x3FFFFFFF,
0x7FFFFFFF,
};
private static uint* BIT_mask =>
(uint*)
System.Runtime.CompilerServices.Unsafe.AsPointer(
ref MemoryMarshal.GetReference(Span_BIT_mask)
);
#else
private static readonly uint* BIT_mask = GetArrayPointer(
new uint[32]
{
0,
1,
3,
7,
0xF,
0x1F,
0x3F,
0x7F,
0xFF,
0x1FF,
0x3FF,
0x7FF,
0xFFF,
0x1FFF,
0x3FFF,
0x7FFF,
0xFFFF,
0x1FFFF,
0x3FFFF,
0x7FFFF,
0xFFFFF,
0x1FFFFF,
0x3FFFFF,
0x7FFFFF,
0xFFFFFF,
0x1FFFFFF,
0x3FFFFFF,
0x7FFFFFF,
0xFFFFFFF,
0x1FFFFFFF,
0x3FFFFFFF,
0x7FFFFFFF,
}
);
#endif
/*-**************************************************************
* bitStream encoding
****************************************************************/
/*! BIT_initCStream() :
* `dstCapacity` must be > sizeof(size_t)
* @return : 0 if success,
* otherwise an error code (can be tested using ERR_isError()) */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initCStream(ref BIT_CStream_t bitC, void* startPtr, nuint dstCapacity)
{
bitC.bitContainer = 0;
bitC.bitPos = 0;
bitC.startPtr = (sbyte*)startPtr;
bitC.ptr = bitC.startPtr;
bitC.endPtr = bitC.startPtr + dstCapacity - sizeof(nuint);
if (dstCapacity <= (nuint)sizeof(nuint))
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall));
return 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getLowerBits(nuint bitContainer, uint nbBits)
{
assert(nbBits < sizeof(uint) * 32 / sizeof(uint));
#if NETCOREAPP3_1_OR_GREATER
if (Bmi2.X64.IsSupported)
{
return (nuint)Bmi2.X64.ZeroHighBits(bitContainer, nbBits);
}
if (Bmi2.IsSupported)
{
return Bmi2.ZeroHighBits((uint)bitContainer, nbBits);
}
#endif
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_addBits() :
* can add up to 31 bits into `bitC`.
* Note : does not check for register overflow ! */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBits(
ref nuint bitC_bitContainer,
ref uint bitC_bitPos,
nuint value,
uint nbBits
)
{
assert(nbBits < sizeof(uint) * 32 / sizeof(uint));
assert(nbBits + bitC_bitPos < (uint)(sizeof(nuint) * 8));
bitC_bitContainer |= BIT_getLowerBits(value, nbBits) << (int)bitC_bitPos;
bitC_bitPos += nbBits;
}
/*! BIT_addBitsFast() :
* works only if `value` is _clean_,
* meaning all high bits above nbBits are 0 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBitsFast(
ref nuint bitC_bitContainer,
ref uint bitC_bitPos,
nuint value,
uint nbBits
)
{
assert(value >> (int)nbBits == 0);
assert(nbBits + bitC_bitPos < (uint)(sizeof(nuint) * 8));
bitC_bitContainer |= value << (int)bitC_bitPos;
bitC_bitPos += nbBits;
}
/*! BIT_flushBitsFast() :
* assumption : bitContainer has not overflowed
* unsafe version; does not check buffer overflow */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBitsFast(
ref nuint bitC_bitContainer,
ref uint bitC_bitPos,
ref sbyte* bitC_ptr,
sbyte* bitC_endPtr
)
{
nuint nbBytes = bitC_bitPos >> 3;
assert(bitC_bitPos < (uint)(sizeof(nuint) * 8));
assert(bitC_ptr <= bitC_endPtr);
MEM_writeLEST(bitC_ptr, bitC_bitContainer);
bitC_ptr += nbBytes;
bitC_bitPos &= 7;
bitC_bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_flushBits() :
* assumption : bitContainer has not overflowed
* safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow.
* overflow will be revealed later on using BIT_closeCStream() */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBits(
ref nuint bitC_bitContainer,
ref uint bitC_bitPos,
ref sbyte* bitC_ptr,
sbyte* bitC_endPtr
)
{
nuint nbBytes = bitC_bitPos >> 3;
assert(bitC_bitPos < (uint)(sizeof(nuint) * 8));
assert(bitC_ptr <= bitC_endPtr);
MEM_writeLEST(bitC_ptr, bitC_bitContainer);
bitC_ptr += nbBytes;
if (bitC_ptr > bitC_endPtr)
bitC_ptr = bitC_endPtr;
bitC_bitPos &= 7;
bitC_bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_closeCStream() :
* @return : size of CStream, in bytes,
* or 0 if it could not fit into dstBuffer */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_closeCStream(
ref nuint bitC_bitContainer,
ref uint bitC_bitPos,
sbyte* bitC_ptr,
sbyte* bitC_endPtr,
sbyte* bitC_startPtr
)
{
BIT_addBitsFast(ref bitC_bitContainer, ref bitC_bitPos, 1, 1);
BIT_flushBits(ref bitC_bitContainer, ref bitC_bitPos, ref bitC_ptr, bitC_endPtr);
if (bitC_ptr >= bitC_endPtr)
return 0;
return (nuint)(bitC_ptr - bitC_startPtr) + (nuint)(bitC_bitPos > 0 ? 1 : 0);
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize), or an errorCode if a problem is detected
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initDStream(BIT_DStream_t* bitD, void* srcBuffer, nuint srcSize)
{
if (srcSize < 1)
{
*bitD = new BIT_DStream_t();
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
bitD->start = (sbyte*)srcBuffer;
bitD->limitPtr = bitD->start + sizeof(nuint);
if (srcSize >= (nuint)sizeof(nuint))
{
bitD->ptr = (sbyte*)srcBuffer + srcSize - sizeof(nuint);
bitD->bitContainer = MEM_readLEST(bitD->ptr);
{
byte lastByte = ((byte*)srcBuffer)[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - ZSTD_highbit32(lastByte) : 0;
if (lastByte == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC));
}
}
else
{
bitD->ptr = bitD->start;
bitD->bitContainer = *(byte*)bitD->start;
switch (srcSize)
{
case 7:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[6] << sizeof(nuint) * 8 - 16;
goto case 6;
case 6:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[5] << sizeof(nuint) * 8 - 24;
goto case 5;
case 5:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[4] << sizeof(nuint) * 8 - 32;
goto case 4;
case 4:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[3] << 24;
goto case 3;
case 3:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[2] << 16;
goto case 2;
case 2:
bitD->bitContainer += (nuint)((byte*)srcBuffer)[1] << 8;
goto default;
default:
break;
}
{
byte lastByte = ((byte*)srcBuffer)[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - ZSTD_highbit32(lastByte) : 0;
if (lastByte == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
}
bitD->bitsConsumed += (uint)((nuint)sizeof(nuint) - srcSize) * 8;
}
return srcSize;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getUpperBits(nuint bitContainer, uint start)
{
return bitContainer >> (int)start;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getMiddleBits(nuint bitContainer, uint start, uint nbBits)
{
uint regMask = (uint)(sizeof(nuint) * 8 - 1);
assert(nbBits < sizeof(uint) * 32 / sizeof(uint));
#if NETCOREAPP3_1_OR_GREATER
if (Bmi2.X64.IsSupported)
{
return (nuint)Bmi2.X64.ZeroHighBits(bitContainer >> (int)(start & regMask), nbBits);
}
if (Bmi2.IsSupported)
{
return Bmi2.ZeroHighBits((uint)(bitContainer >> (int)(start & regMask)), nbBits);
}
#endif
return (nuint)(bitContainer >> (int)(start & regMask) & ((ulong)1 << (int)nbBits) - 1);
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBits(BIT_DStream_t* bitD, uint nbBits)
{
return BIT_getMiddleBits(
bitD->bitContainer,
(uint)(sizeof(nuint) * 8) - bitD->bitsConsumed - nbBits,
nbBits
);
}
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
uint regMask = (uint)(sizeof(nuint) * 8 - 1);
assert(nbBits >= 1);
return bitD->bitContainer
<< (int)(bitD->bitsConsumed & regMask)
>> (int)(regMask + 1 - nbBits & regMask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_skipBits(BIT_DStream_t* bitD, uint nbBits)
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBits(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_reloadDStream_internal() :
* Simple variant of BIT_reloadDStream(), with two conditions:
* 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
* 2. look window is valid after shifted down : bitD->ptr >= bitD->start
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
{
assert(bitD->bitsConsumed <= (uint)(sizeof(nuint) * 8));
bitD->ptr -= bitD->bitsConsumed >> 3;
assert(bitD->ptr >= bitD->start);
bitD->bitsConsumed &= 7;
bitD->bitContainer = MEM_readLEST(bitD->ptr);
return BIT_DStream_status.BIT_DStream_unfinished;
}
/*! BIT_reloadDStreamFast() :
* Similar to BIT_reloadDStream(), but with two differences:
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
* point you must use BIT_reloadDStream() to reload.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
{
if (bitD->ptr < bitD->limitPtr)
return BIT_DStream_status.BIT_DStream_overflow;
return BIT_reloadDStream_internal(bitD);
}
#if NET7_0_OR_GREATER
private static ReadOnlySpan<byte> Span_static_zeroFilled =>
new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 };
private static nuint* static_zeroFilled =>
(nuint*)
System.Runtime.CompilerServices.Unsafe.AsPointer(
ref MemoryMarshal.GetReference(Span_static_zeroFilled)
);
#else
private static readonly nuint* static_zeroFilled = (nuint*)GetArrayPointer(
new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }
);
#endif
/*! BIT_reloadDStream() :
* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not never beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > (uint)(sizeof(nuint) * 8))
{
bitD->ptr = (sbyte*)&static_zeroFilled[0];
return BIT_DStream_status.BIT_DStream_overflow;
}
assert(bitD->ptr >= bitD->start);
if (bitD->ptr >= bitD->limitPtr)
{
return BIT_reloadDStream_internal(bitD);
}
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < (uint)(sizeof(nuint) * 8))
return BIT_DStream_status.BIT_DStream_endOfBuffer;
return BIT_DStream_status.BIT_DStream_completed;
}
{
uint nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_status.BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (uint)(bitD->ptr - bitD->start);
result = BIT_DStream_status.BIT_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes * 8;
bitD->bitContainer = MEM_readLEST(bitD->ptr);
return result;
}
}
/*! BIT_endOfDStream() :
* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint BIT_endOfDStream(BIT_DStream_t* DStream)
{
return DStream->ptr == DStream->start && DStream->bitsConsumed == (uint)(sizeof(nuint) * 8)
? 1U
: 0U;
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize), or an errorCode if a problem is detected
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initDStream(ref BIT_DStream_t bitD, void* srcBuffer, nuint srcSize)
{
if (srcSize < 1)
{
bitD = new BIT_DStream_t();
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
bitD.start = (sbyte*)srcBuffer;
bitD.limitPtr = bitD.start + sizeof(nuint);
if (srcSize >= (nuint)sizeof(nuint))
{
bitD.ptr = (sbyte*)srcBuffer + srcSize - sizeof(nuint);
bitD.bitContainer = MEM_readLEST(bitD.ptr);
{
byte lastByte = ((byte*)srcBuffer)[srcSize - 1];
bitD.bitsConsumed = lastByte != 0 ? 8 - ZSTD_highbit32(lastByte) : 0;
if (lastByte == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC));
}
}
else
{
bitD.ptr = bitD.start;
bitD.bitContainer = *(byte*)bitD.start;
switch (srcSize)
{
case 7:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[6] << sizeof(nuint) * 8 - 16;
goto case 6;
case 6:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[5] << sizeof(nuint) * 8 - 24;
goto case 5;
case 5:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[4] << sizeof(nuint) * 8 - 32;
goto case 4;
case 4:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[3] << 24;
goto case 3;
case 3:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[2] << 16;
goto case 2;
case 2:
bitD.bitContainer += (nuint)((byte*)srcBuffer)[1] << 8;
goto default;
default:
break;
}
{
byte lastByte = ((byte*)srcBuffer)[srcSize - 1];
bitD.bitsConsumed = lastByte != 0 ? 8 - ZSTD_highbit32(lastByte) : 0;
if (lastByte == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
}
bitD.bitsConsumed += (uint)((nuint)sizeof(nuint) - srcSize) * 8;
}
return srcSize;
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBits(nuint bitD_bitContainer, uint bitD_bitsConsumed, uint nbBits)
{
return BIT_getMiddleBits(
bitD_bitContainer,
(uint)(sizeof(nuint) * 8) - bitD_bitsConsumed - nbBits,
nbBits
);
}
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBitsFast(
nuint bitD_bitContainer,
uint bitD_bitsConsumed,
uint nbBits
)
{
uint regMask = (uint)(sizeof(nuint) * 8 - 1);
assert(nbBits >= 1);
return bitD_bitContainer
<< (int)(bitD_bitsConsumed & regMask)
>> (int)(regMask + 1 - nbBits & regMask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_skipBits(ref uint bitD_bitsConsumed, uint nbBits)
{
bitD_bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBits(
nuint bitD_bitContainer,
ref uint bitD_bitsConsumed,
uint nbBits
)
{
nuint value = BIT_lookBits(bitD_bitContainer, bitD_bitsConsumed, nbBits);
BIT_skipBits(ref bitD_bitsConsumed, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBitsFast(
nuint bitD_bitContainer,
ref uint bitD_bitsConsumed,
uint nbBits
)
{
nuint value = BIT_lookBitsFast(bitD_bitContainer, bitD_bitsConsumed, nbBits);
assert(nbBits >= 1);
BIT_skipBits(ref bitD_bitsConsumed, nbBits);
return value;
}
/*! BIT_reloadDStreamFast() :
* Similar to BIT_reloadDStream(), but with two differences:
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
* point you must use BIT_reloadDStream() to reload.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStreamFast(
ref nuint bitD_bitContainer,
ref uint bitD_bitsConsumed,
ref sbyte* bitD_ptr,
sbyte* bitD_start,
sbyte* bitD_limitPtr
)
{
if (bitD_ptr < bitD_limitPtr)
return BIT_DStream_status.BIT_DStream_overflow;
return BIT_reloadDStream_internal(
ref bitD_bitContainer,
ref bitD_bitsConsumed,
ref bitD_ptr,
bitD_start
);
}
/*! BIT_reloadDStream() :
* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not never beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream(
ref nuint bitD_bitContainer,
ref uint bitD_bitsConsumed,
ref sbyte* bitD_ptr,
sbyte* bitD_start,
sbyte* bitD_limitPtr
)
{
if (bitD_bitsConsumed > (uint)(sizeof(nuint) * 8))
{
bitD_ptr = (sbyte*)&static_zeroFilled[0];
return BIT_DStream_status.BIT_DStream_overflow;
}
assert(bitD_ptr >= bitD_start);
if (bitD_ptr >= bitD_limitPtr)
{
return BIT_reloadDStream_internal(
ref bitD_bitContainer,
ref bitD_bitsConsumed,
ref bitD_ptr,
bitD_start
);
}
if (bitD_ptr == bitD_start)
{
if (bitD_bitsConsumed < (uint)(sizeof(nuint) * 8))
return BIT_DStream_status.BIT_DStream_endOfBuffer;
return BIT_DStream_status.BIT_DStream_completed;
}
{
uint nbBytes = bitD_bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_status.BIT_DStream_unfinished;
if (bitD_ptr - nbBytes < bitD_start)
{
nbBytes = (uint)(bitD_ptr - bitD_start);
result = BIT_DStream_status.BIT_DStream_endOfBuffer;
}
bitD_ptr -= nbBytes;
bitD_bitsConsumed -= nbBytes * 8;
bitD_bitContainer = MEM_readLEST(bitD_ptr);
return result;
}
}
/*! BIT_reloadDStream_internal() :
* Simple variant of BIT_reloadDStream(), with two conditions:
* 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
* 2. look window is valid after shifted down : bitD->ptr >= bitD->start
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream_internal(
ref nuint bitD_bitContainer,
ref uint bitD_bitsConsumed,
ref sbyte* bitD_ptr,
sbyte* bitD_start
)
{
assert(bitD_bitsConsumed <= (uint)(sizeof(nuint) * 8));
bitD_ptr -= bitD_bitsConsumed >> 3;
assert(bitD_ptr >= bitD_start);
bitD_bitsConsumed &= 7;
bitD_bitContainer = MEM_readLEST(bitD_ptr);
return BIT_DStream_status.BIT_DStream_unfinished;
}
/*! BIT_endOfDStream() :
* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint BIT_endOfDStream(
uint DStream_bitsConsumed,
sbyte* DStream_ptr,
sbyte* DStream_start
)
{
return DStream_ptr == DStream_start && DStream_bitsConsumed == (uint)(sizeof(nuint) * 8)
? 1U
: 0U;
}
}

View File

@@ -1,8 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public struct BlockSummary
{
public nuint nbSequences;
public nuint blockSize;
public nuint litSize;
}

View File

@@ -1,20 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
* COVER_best_t is used for two purposes:
* 1. Synchronizing threads.
* 2. Saving the best parameters and dictionary.
*
* All of the methods except COVER_best_init() are thread safe if zstd is
* compiled with multithreaded support.
*/
public unsafe struct COVER_best_s
{
public void* mutex;
public void* cond;
public nuint liveJobs;
public void* dict;
public nuint dictSize;
public ZDICT_cover_params_t parameters;
public nuint compressedSize;
}

View File

@@ -1,19 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/*-*************************************
* Context
***************************************/
public unsafe struct COVER_ctx_t
{
public byte* samples;
public nuint* offsets;
public nuint* samplesSizes;
public nuint nbSamples;
public nuint nbTrainSamples;
public nuint nbTestSamples;
public uint* suffix;
public nuint suffixSize;
public uint* freqs;
public uint* dmerAt;
public uint d;
}

View File

@@ -1,11 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
* Struct used for the dictionary selection function.
*/
public unsafe struct COVER_dictSelection
{
public byte* dictContent;
public nuint dictSize;
public nuint totalCompressedSize;
}

View File

@@ -1,10 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
*Number of epochs and size of each epoch.
*/
public struct COVER_epoch_info_t
{
public uint num;
public uint size;
}

View File

@@ -1,7 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public struct COVER_map_pair_t_s
{
public uint key;
public uint value;
}

View File

@@ -1,9 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public unsafe struct COVER_map_s
{
public COVER_map_pair_t_s* data;
public uint sizeLog;
public uint size;
public uint sizeMask;
}

View File

@@ -1,11 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
* A segment is a range in the source as well as the score of the segment.
*/
public struct COVER_segment_t
{
public uint begin;
public uint end;
public uint score;
}

View File

@@ -1,12 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
* Parameters for COVER_tryParameters().
*/
public unsafe struct COVER_tryParameters_data_s
{
public COVER_ctx_t* ctx;
public COVER_best_s* best;
public nuint dictBufferCapacity;
public ZDICT_cover_params_t parameters;
}

View File

@@ -1,849 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
private static readonly ZSTD_compressionParameters[][] ZSTD_defaultCParameters =
new ZSTD_compressionParameters[4][]
{
new ZSTD_compressionParameters[23]
{
new ZSTD_compressionParameters(
windowLog: 19,
chainLog: 12,
hashLog: 13,
searchLog: 1,
minMatch: 6,
targetLength: 1,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 19,
chainLog: 13,
hashLog: 14,
searchLog: 1,
minMatch: 7,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 20,
chainLog: 15,
hashLog: 16,
searchLog: 1,
minMatch: 6,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 16,
hashLog: 17,
searchLog: 1,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 18,
hashLog: 18,
searchLog: 1,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 18,
hashLog: 19,
searchLog: 3,
minMatch: 5,
targetLength: 2,
strategy: ZSTD_strategy.ZSTD_greedy
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 18,
hashLog: 19,
searchLog: 3,
minMatch: 5,
targetLength: 4,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 19,
hashLog: 20,
searchLog: 4,
minMatch: 5,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 21,
chainLog: 19,
hashLog: 20,
searchLog: 4,
minMatch: 5,
targetLength: 16,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 20,
hashLog: 21,
searchLog: 4,
minMatch: 5,
targetLength: 16,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 21,
hashLog: 22,
searchLog: 5,
minMatch: 5,
targetLength: 16,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 21,
hashLog: 22,
searchLog: 6,
minMatch: 5,
targetLength: 16,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 22,
hashLog: 23,
searchLog: 6,
minMatch: 5,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 22,
hashLog: 22,
searchLog: 4,
minMatch: 5,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 22,
hashLog: 23,
searchLog: 5,
minMatch: 5,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 23,
hashLog: 23,
searchLog: 6,
minMatch: 5,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 22,
chainLog: 22,
hashLog: 22,
searchLog: 5,
minMatch: 5,
targetLength: 48,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 23,
chainLog: 23,
hashLog: 22,
searchLog: 5,
minMatch: 4,
targetLength: 64,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 23,
chainLog: 23,
hashLog: 22,
searchLog: 6,
minMatch: 3,
targetLength: 64,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 23,
chainLog: 24,
hashLog: 22,
searchLog: 7,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 25,
chainLog: 25,
hashLog: 23,
searchLog: 7,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 26,
chainLog: 26,
hashLog: 24,
searchLog: 7,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 27,
chainLog: 27,
hashLog: 25,
searchLog: 9,
minMatch: 3,
targetLength: 999,
strategy: ZSTD_strategy.ZSTD_btultra2
),
},
new ZSTD_compressionParameters[23]
{
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 12,
hashLog: 13,
searchLog: 1,
minMatch: 5,
targetLength: 1,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 13,
hashLog: 14,
searchLog: 1,
minMatch: 6,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 14,
hashLog: 14,
searchLog: 1,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 16,
hashLog: 16,
searchLog: 1,
minMatch: 4,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 16,
hashLog: 17,
searchLog: 3,
minMatch: 5,
targetLength: 2,
strategy: ZSTD_strategy.ZSTD_greedy
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 17,
hashLog: 18,
searchLog: 5,
minMatch: 5,
targetLength: 2,
strategy: ZSTD_strategy.ZSTD_greedy
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 3,
minMatch: 5,
targetLength: 4,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 4,
minMatch: 4,
targetLength: 4,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 4,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 5,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 6,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 5,
minMatch: 4,
targetLength: 12,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 7,
minMatch: 4,
targetLength: 12,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 4,
minMatch: 4,
targetLength: 16,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 4,
minMatch: 3,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 18,
hashLog: 19,
searchLog: 6,
minMatch: 3,
targetLength: 128,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 6,
minMatch: 3,
targetLength: 128,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 8,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 6,
minMatch: 3,
targetLength: 128,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 8,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 10,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 12,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 18,
chainLog: 19,
hashLog: 19,
searchLog: 13,
minMatch: 3,
targetLength: 999,
strategy: ZSTD_strategy.ZSTD_btultra2
),
},
new ZSTD_compressionParameters[23]
{
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 12,
hashLog: 12,
searchLog: 1,
minMatch: 5,
targetLength: 1,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 12,
hashLog: 13,
searchLog: 1,
minMatch: 6,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 13,
hashLog: 15,
searchLog: 1,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 15,
hashLog: 16,
searchLog: 2,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 17,
hashLog: 17,
searchLog: 2,
minMatch: 4,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 3,
minMatch: 4,
targetLength: 2,
strategy: ZSTD_strategy.ZSTD_greedy
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 3,
minMatch: 4,
targetLength: 4,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 3,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 4,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 5,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 16,
hashLog: 17,
searchLog: 6,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 17,
hashLog: 17,
searchLog: 5,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 7,
minMatch: 4,
targetLength: 12,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 3,
minMatch: 4,
targetLength: 12,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 4,
minMatch: 3,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 6,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 6,
minMatch: 3,
targetLength: 128,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 8,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 10,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 5,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 7,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 9,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 17,
chainLog: 18,
hashLog: 17,
searchLog: 11,
minMatch: 3,
targetLength: 999,
strategy: ZSTD_strategy.ZSTD_btultra2
),
},
new ZSTD_compressionParameters[23]
{
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 12,
hashLog: 13,
searchLog: 1,
minMatch: 5,
targetLength: 1,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 15,
searchLog: 1,
minMatch: 5,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 15,
searchLog: 1,
minMatch: 4,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_fast
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 15,
searchLog: 2,
minMatch: 4,
targetLength: 0,
strategy: ZSTD_strategy.ZSTD_dfast
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 14,
searchLog: 4,
minMatch: 4,
targetLength: 2,
strategy: ZSTD_strategy.ZSTD_greedy
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 14,
searchLog: 3,
minMatch: 4,
targetLength: 4,
strategy: ZSTD_strategy.ZSTD_lazy
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 14,
searchLog: 4,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 14,
searchLog: 6,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 14,
hashLog: 14,
searchLog: 8,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_lazy2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 14,
searchLog: 5,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 14,
searchLog: 9,
minMatch: 4,
targetLength: 8,
strategy: ZSTD_strategy.ZSTD_btlazy2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 14,
searchLog: 3,
minMatch: 4,
targetLength: 12,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 14,
searchLog: 4,
minMatch: 3,
targetLength: 24,
strategy: ZSTD_strategy.ZSTD_btopt
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 14,
searchLog: 5,
minMatch: 3,
targetLength: 32,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 6,
minMatch: 3,
targetLength: 64,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 7,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 5,
minMatch: 3,
targetLength: 48,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 6,
minMatch: 3,
targetLength: 128,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 7,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 8,
minMatch: 3,
targetLength: 256,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 8,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 9,
minMatch: 3,
targetLength: 512,
strategy: ZSTD_strategy.ZSTD_btultra2
),
new ZSTD_compressionParameters(
windowLog: 14,
chainLog: 15,
hashLog: 15,
searchLog: 10,
minMatch: 3,
targetLength: 999,
strategy: ZSTD_strategy.ZSTD_btultra2
),
},
};
}

View File

@@ -1,61 +0,0 @@
using System.Runtime.CompilerServices;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
/* @return 1 if @u is a 2^n value, 0 otherwise
* useful to check a value is valid for alignment restrictions */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ZSTD_isPower2(nuint u)
{
return (u & u - 1) == 0 ? 1 : 0;
}
/**
* Helper function to perform a wrapped pointer difference without triggering
* UBSAN.
*
* @returns lhs - rhs with wrapping
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nint ZSTD_wrappedPtrDiff(byte* lhs, byte* rhs)
{
return (nint)(lhs - rhs);
}
/**
* Helper function to perform a wrapped pointer add without triggering UBSAN.
*
* @return ptr + add with wrapping
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte* ZSTD_wrappedPtrAdd(byte* ptr, nint add)
{
return ptr + add;
}
/**
* Helper function to perform a wrapped pointer subtraction without triggering
* UBSAN.
*
* @return ptr - sub with wrapping
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte* ZSTD_wrappedPtrSub(byte* ptr, nint sub)
{
return ptr - sub;
}
/**
* Helper function to add to a pointer that works around C's undefined behavior
* of adding 0 to NULL.
*
* @returns `ptr + add` except it defines `NULL + 0 == NULL`.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte* ZSTD_maybeNullPtrAdd(byte* ptr, nint add)
{
return add > 0 ? ptr + add : ptr;
}
}

View File

@@ -1,444 +0,0 @@
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
private static int g_displayLevel = 0;
/**
* Returns the sum of the sample sizes.
*/
private static nuint COVER_sum(nuint* samplesSizes, uint nbSamples)
{
nuint sum = 0;
uint i;
for (i = 0; i < nbSamples; ++i)
{
sum += samplesSizes[i];
}
return sum;
}
/**
* Warns the user when their corpus is too small.
*/
private static void COVER_warnOnSmallCorpus(nuint maxDictSize, nuint nbDmers, int displayLevel)
{
double ratio = nbDmers / (double)maxDictSize;
if (ratio >= 10)
{
return;
}
}
/**
* Computes the number of epochs and the size of each epoch.
* We will make sure that each epoch gets at least 10 * k bytes.
*
* The COVER algorithms divide the data up into epochs of equal size and
* select one segment from each epoch.
*
* @param maxDictSize The maximum allowed dictionary size.
* @param nbDmers The number of dmers we are training on.
* @param k The parameter k (segment size).
* @param passes The target number of passes over the dmer corpus.
* More passes means a better dictionary.
*/
private static COVER_epoch_info_t COVER_computeEpochs(
uint maxDictSize,
uint nbDmers,
uint k,
uint passes
)
{
uint minEpochSize = k * 10;
COVER_epoch_info_t epochs;
epochs.num = 1 > maxDictSize / k / passes ? 1 : maxDictSize / k / passes;
epochs.size = nbDmers / epochs.num;
if (epochs.size >= minEpochSize)
{
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
epochs.size = minEpochSize < nbDmers ? minEpochSize : nbDmers;
epochs.num = nbDmers / epochs.size;
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
/**
* Checks total compressed size of a dictionary
*/
private static nuint COVER_checkTotalCompressedSize(
ZDICT_cover_params_t parameters,
nuint* samplesSizes,
byte* samples,
nuint* offsets,
nuint nbTrainSamples,
nuint nbSamples,
byte* dict,
nuint dictBufferCapacity
)
{
nuint totalCompressedSize = unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC));
/* Pointers */
ZSTD_CCtx_s* cctx;
ZSTD_CDict_s* cdict;
void* dst;
/* Local variables */
nuint dstCapacity;
nuint i;
{
nuint maxSampleSize = 0;
i = parameters.splitPoint < 1 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
maxSampleSize = samplesSizes[i] > maxSampleSize ? samplesSizes[i] : maxSampleSize;
}
dstCapacity = ZSTD_compressBound(maxSampleSize);
dst = malloc(dstCapacity);
}
cctx = ZSTD_createCCtx();
cdict = ZSTD_createCDict(dict, dictBufferCapacity, parameters.zParams.compressionLevel);
if (dst == null || cctx == null || cdict == null)
{
goto _compressCleanup;
}
totalCompressedSize = dictBufferCapacity;
i = parameters.splitPoint < 1 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
nuint size = ZSTD_compress_usingCDict(
cctx,
dst,
dstCapacity,
samples + offsets[i],
samplesSizes[i],
cdict
);
if (ERR_isError(size))
{
totalCompressedSize = size;
goto _compressCleanup;
}
totalCompressedSize += size;
}
_compressCleanup:
ZSTD_freeCCtx(cctx);
ZSTD_freeCDict(cdict);
if (dst != null)
{
free(dst);
}
return totalCompressedSize;
}
/**
* Initialize the `COVER_best_t`.
*/
private static void COVER_best_init(COVER_best_s* best)
{
if (best == null)
return;
SynchronizationWrapper.Init(&best->mutex);
best->liveJobs = 0;
best->dict = null;
best->dictSize = 0;
best->compressedSize = unchecked((nuint)(-1));
best->parameters = new ZDICT_cover_params_t();
}
/**
* Wait until liveJobs == 0.
*/
private static void COVER_best_wait(COVER_best_s* best)
{
if (best == null)
{
return;
}
SynchronizationWrapper.Enter(&best->mutex);
while (best->liveJobs != 0)
{
SynchronizationWrapper.Wait(&best->mutex);
}
SynchronizationWrapper.Exit(&best->mutex);
}
/**
* Call COVER_best_wait() and then destroy the COVER_best_t.
*/
private static void COVER_best_destroy(COVER_best_s* best)
{
if (best == null)
{
return;
}
COVER_best_wait(best);
if (best->dict != null)
{
free(best->dict);
}
SynchronizationWrapper.Free(&best->mutex);
}
/**
* Called when a thread is about to be launched.
* Increments liveJobs.
*/
private static void COVER_best_start(COVER_best_s* best)
{
if (best == null)
{
return;
}
SynchronizationWrapper.Enter(&best->mutex);
++best->liveJobs;
SynchronizationWrapper.Exit(&best->mutex);
}
/**
* Called when a thread finishes executing, both on error or success.
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
private static void COVER_best_finish(
COVER_best_s* best,
ZDICT_cover_params_t parameters,
COVER_dictSelection selection
)
{
void* dict = selection.dictContent;
nuint compressedSize = selection.totalCompressedSize;
nuint dictSize = selection.dictSize;
if (best == null)
{
return;
}
{
nuint liveJobs;
SynchronizationWrapper.Enter(&best->mutex);
--best->liveJobs;
liveJobs = best->liveJobs;
if (compressedSize < best->compressedSize)
{
if (best->dict == null || best->dictSize < dictSize)
{
if (best->dict != null)
{
free(best->dict);
}
best->dict = malloc(dictSize);
if (best->dict == null)
{
best->compressedSize = unchecked(
(nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)
);
best->dictSize = 0;
SynchronizationWrapper.Pulse(&best->mutex);
SynchronizationWrapper.Exit(&best->mutex);
return;
}
}
if (dict != null)
{
memcpy(best->dict, dict, (uint)dictSize);
best->dictSize = dictSize;
best->parameters = parameters;
best->compressedSize = compressedSize;
}
}
if (liveJobs == 0)
{
SynchronizationWrapper.PulseAll(&best->mutex);
}
SynchronizationWrapper.Exit(&best->mutex);
}
}
private static COVER_dictSelection setDictSelection(byte* buf, nuint s, nuint csz)
{
COVER_dictSelection ds;
ds.dictContent = buf;
ds.dictSize = s;
ds.totalCompressedSize = csz;
return ds;
}
/**
* Error function for COVER_selectDict function. Returns a struct where
* return.totalCompressedSize is a ZSTD error.
*/
private static COVER_dictSelection COVER_dictSelectionError(nuint error)
{
return setDictSelection(null, 0, error);
}
/**
* Error function for COVER_selectDict function. Checks if the return
* value is an error.
*/
private static uint COVER_dictSelectionIsError(COVER_dictSelection selection)
{
return ERR_isError(selection.totalCompressedSize) || selection.dictContent == null
? 1U
: 0U;
}
/**
* Always call after selectDict is called to free up used memory from
* newly created dictionary.
*/
private static void COVER_dictSelectionFree(COVER_dictSelection selection)
{
free(selection.dictContent);
}
/**
* Called to finalize the dictionary and select one based on whether or not
* the shrink-dict flag was enabled. If enabled the dictionary used is the
* smallest dictionary within a specified regression of the compressed size
* from the largest dictionary.
*/
private static COVER_dictSelection COVER_selectDict(
byte* customDictContent,
nuint dictBufferCapacity,
nuint dictContentSize,
byte* samplesBuffer,
nuint* samplesSizes,
uint nbFinalizeSamples,
nuint nbCheckSamples,
nuint nbSamples,
ZDICT_cover_params_t @params,
nuint* offsets,
nuint totalCompressedSize
)
{
nuint largestDict = 0;
nuint largestCompressed = 0;
byte* customDictContentEnd = customDictContent + dictContentSize;
byte* largestDictbuffer = (byte*)malloc(dictBufferCapacity);
byte* candidateDictBuffer = (byte*)malloc(dictBufferCapacity);
double regressionTolerance = (double)@params.shrinkDictMaxRegression / 100 + 1;
if (largestDictbuffer == null || candidateDictBuffer == null)
{
free(largestDictbuffer);
free(candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
memcpy(largestDictbuffer, customDictContent, (uint)dictContentSize);
dictContentSize = ZDICT_finalizeDictionary(
largestDictbuffer,
dictBufferCapacity,
customDictContent,
dictContentSize,
samplesBuffer,
samplesSizes,
nbFinalizeSamples,
@params.zParams
);
if (ZDICT_isError(dictContentSize))
{
free(largestDictbuffer);
free(candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(
@params,
samplesSizes,
samplesBuffer,
offsets,
nbCheckSamples,
nbSamples,
largestDictbuffer,
dictContentSize
);
if (ERR_isError(totalCompressedSize))
{
free(largestDictbuffer);
free(candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (@params.shrinkDict == 0)
{
free(candidateDictBuffer);
return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
}
largestDict = dictContentSize;
largestCompressed = totalCompressedSize;
dictContentSize = 256;
while (dictContentSize < largestDict)
{
memcpy(candidateDictBuffer, largestDictbuffer, (uint)largestDict);
dictContentSize = ZDICT_finalizeDictionary(
candidateDictBuffer,
dictBufferCapacity,
customDictContentEnd - dictContentSize,
dictContentSize,
samplesBuffer,
samplesSizes,
nbFinalizeSamples,
@params.zParams
);
if (ZDICT_isError(dictContentSize))
{
free(largestDictbuffer);
free(candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(
@params,
samplesSizes,
samplesBuffer,
offsets,
nbCheckSamples,
nbSamples,
candidateDictBuffer,
dictContentSize
);
if (ERR_isError(totalCompressedSize))
{
free(largestDictbuffer);
free(candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (totalCompressedSize <= largestCompressed * regressionTolerance)
{
free(largestDictbuffer);
return setDictSelection(candidateDictBuffer, dictContentSize, totalCompressedSize);
}
dictContentSize *= 2;
}
dictContentSize = largestDict;
totalCompressedSize = largestCompressed;
free(candidateDictBuffer);
return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
}
}

View File

@@ -1,12 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/*-***************************/
/* generic DTableDesc */
/*-***************************/
public struct DTableDesc
{
public byte maxTableLog;
public byte tableType;
public byte tableLog;
public byte reserved;
}

View File

@@ -1,13 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public unsafe struct EStats_ress_t
{
/* dictionary */
public ZSTD_CDict_s* dict;
/* working context */
public ZSTD_CCtx_s* zc;
/* must be ZSTD_BLOCKSIZE_MAX allocated */
public void* workPlace;
}

View File

@@ -1,447 +0,0 @@
using System.Runtime.CompilerServices;
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
/*=== Version ===*/
private static uint FSE_versionNumber()
{
return 0 * 100 * 100 + 9 * 100 + 0;
}
/*=== Error Management ===*/
private static bool FSE_isError(nuint code)
{
return ERR_isError(code);
}
private static string FSE_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/* Error Management */
private static bool HUF_isError(nuint code)
{
return ERR_isError(code);
}
private static string HUF_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_readNCount_body(
short* normalizedCounter,
uint* maxSVPtr,
uint* tableLogPtr,
void* headerBuffer,
nuint hbSize
)
{
byte* istart = (byte*)headerBuffer;
byte* iend = istart + hbSize;
byte* ip = istart;
int nbBits;
int remaining;
int threshold;
uint bitStream;
int bitCount;
uint charnum = 0;
uint maxSV1 = *maxSVPtr + 1;
int previous0 = 0;
if (hbSize < 8)
{
sbyte* buffer = stackalloc sbyte[8];
/* This function only works when hbSize >= 8 */
memset(buffer, 0, sizeof(sbyte) * 8);
memcpy(buffer, headerBuffer, (uint)hbSize);
{
nuint countSize = FSE_readNCount(
normalizedCounter,
maxSVPtr,
tableLogPtr,
buffer,
sizeof(sbyte) * 8
);
if (FSE_isError(countSize))
return countSize;
if (countSize > hbSize)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
return countSize;
}
}
assert(hbSize >= 8);
memset(normalizedCounter, 0, (*maxSVPtr + 1) * sizeof(short));
bitStream = MEM_readLE32(ip);
nbBits = (int)((bitStream & 0xF) + 5);
if (nbBits > 15)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge));
bitStream >>= 4;
bitCount = 4;
*tableLogPtr = (uint)nbBits;
remaining = (1 << nbBits) + 1;
threshold = 1 << nbBits;
nbBits++;
for (; ; )
{
if (previous0 != 0)
{
/* Count the number of repeats. Each time the
* 2-bit repeat code is 0b11 there is another
* repeat.
* Avoid UB by setting the high bit to 1.
*/
int repeats = (int)(ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1);
while (repeats >= 12)
{
charnum += 3 * 12;
if (ip <= iend - 7)
{
ip += 3;
}
else
{
bitCount -= (int)(8 * (iend - 7 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> bitCount;
repeats = (int)(ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1);
}
charnum += (uint)(3 * repeats);
bitStream >>= 2 * repeats;
bitCount += 2 * repeats;
assert((bitStream & 3) < 3);
charnum += bitStream & 3;
bitCount += 2;
if (charnum >= maxSV1)
break;
if (ip <= iend - 7 || ip + (bitCount >> 3) <= iend - 4)
{
assert(bitCount >> 3 <= 3);
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> bitCount;
}
{
int max = 2 * threshold - 1 - remaining;
int count;
if ((bitStream & (uint)(threshold - 1)) < (uint)max)
{
count = (int)(bitStream & (uint)(threshold - 1));
bitCount += nbBits - 1;
}
else
{
count = (int)(bitStream & (uint)(2 * threshold - 1));
if (count >= threshold)
count -= max;
bitCount += nbBits;
}
count--;
if (count >= 0)
{
remaining -= count;
}
else
{
assert(count == -1);
remaining += count;
}
normalizedCounter[charnum++] = (short)count;
previous0 = count == 0 ? 1 : 0;
assert(threshold > 1);
if (remaining < threshold)
{
if (remaining <= 1)
break;
nbBits = (int)(ZSTD_highbit32((uint)remaining) + 1);
threshold = 1 << nbBits - 1;
}
if (charnum >= maxSV1)
break;
if (ip <= iend - 7 || ip + (bitCount >> 3) <= iend - 4)
{
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> bitCount;
}
}
if (remaining != 1)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
if (charnum > maxSV1)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall));
if (bitCount > 32)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
*maxSVPtr = charnum - 1;
ip += bitCount + 7 >> 3;
return (nuint)(ip - istart);
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint FSE_readNCount_body_default(
short* normalizedCounter,
uint* maxSVPtr,
uint* tableLogPtr,
void* headerBuffer,
nuint hbSize
)
{
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
/*! FSE_readNCount_bmi2():
* Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
*/
private static nuint FSE_readNCount_bmi2(
short* normalizedCounter,
uint* maxSVPtr,
uint* tableLogPtr,
void* headerBuffer,
nuint hbSize,
int bmi2
)
{
return FSE_readNCount_body_default(
normalizedCounter,
maxSVPtr,
tableLogPtr,
headerBuffer,
hbSize
);
}
/*! FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
private static nuint FSE_readNCount(
short* normalizedCounter,
uint* maxSVPtr,
uint* tableLogPtr,
void* headerBuffer,
nuint hbSize
)
{
return FSE_readNCount_bmi2(
normalizedCounter,
maxSVPtr,
tableLogPtr,
headerBuffer,
hbSize,
0
);
}
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
private static nuint HUF_readStats(
byte* huffWeight,
nuint hwSize,
uint* rankStats,
uint* nbSymbolsPtr,
uint* tableLogPtr,
void* src,
nuint srcSize
)
{
uint* wksp = stackalloc uint[219];
return HUF_readStats_wksp(
huffWeight,
hwSize,
rankStats,
nbSymbolsPtr,
tableLogPtr,
src,
srcSize,
wksp,
sizeof(uint) * 219,
0
);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint HUF_readStats_body(
byte* huffWeight,
nuint hwSize,
uint* rankStats,
uint* nbSymbolsPtr,
uint* tableLogPtr,
void* src,
nuint srcSize,
void* workSpace,
nuint wkspSize,
int bmi2
)
{
uint weightTotal;
byte* ip = (byte*)src;
nuint iSize;
nuint oSize;
if (srcSize == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
iSize = ip[0];
if (iSize >= 128)
{
oSize = iSize - 127;
iSize = (oSize + 1) / 2;
if (iSize + 1 > srcSize)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
if (oSize >= hwSize)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
ip += 1;
{
uint n;
for (n = 0; n < oSize; n += 2)
{
huffWeight[n] = (byte)(ip[n / 2] >> 4);
huffWeight[n + 1] = (byte)(ip[n / 2] & 15);
}
}
}
else
{
if (iSize + 1 > srcSize)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
oSize = FSE_decompress_wksp_bmi2(
huffWeight,
hwSize - 1,
ip + 1,
iSize,
6,
workSpace,
wkspSize,
bmi2
);
if (FSE_isError(oSize))
return oSize;
}
memset(rankStats, 0, (12 + 1) * sizeof(uint));
weightTotal = 0;
{
uint n;
for (n = 0; n < oSize; n++)
{
if (huffWeight[n] > 12)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
rankStats[huffWeight[n]]++;
weightTotal += (uint)(1 << huffWeight[n] >> 1);
}
}
if (weightTotal == 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
{
uint tableLog = ZSTD_highbit32(weightTotal) + 1;
if (tableLog > 12)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
*tableLogPtr = tableLog;
{
uint total = (uint)(1 << (int)tableLog);
uint rest = total - weightTotal;
uint verif = (uint)(1 << (int)ZSTD_highbit32(rest));
uint lastWeight = ZSTD_highbit32(rest) + 1;
if (verif != rest)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
huffWeight[oSize] = (byte)lastWeight;
rankStats[lastWeight]++;
}
}
if (rankStats[1] < 2 || (rankStats[1] & 1) != 0)
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected));
*nbSymbolsPtr = (uint)(oSize + 1);
return iSize + 1;
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint HUF_readStats_body_default(
byte* huffWeight,
nuint hwSize,
uint* rankStats,
uint* nbSymbolsPtr,
uint* tableLogPtr,
void* src,
nuint srcSize,
void* workSpace,
nuint wkspSize
)
{
return HUF_readStats_body(
huffWeight,
hwSize,
rankStats,
nbSymbolsPtr,
tableLogPtr,
src,
srcSize,
workSpace,
wkspSize,
0
);
}
private static nuint HUF_readStats_wksp(
byte* huffWeight,
nuint hwSize,
uint* rankStats,
uint* nbSymbolsPtr,
uint* tableLogPtr,
void* src,
nuint srcSize,
void* workSpace,
nuint wkspSize,
int flags
)
{
return HUF_readStats_body_default(
huffWeight,
hwSize,
rankStats,
nbSymbolsPtr,
tableLogPtr,
src,
srcSize,
workSpace,
wkspSize
);
}
}

View File

@@ -1,110 +0,0 @@
using System.Runtime.CompilerServices;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool ERR_isError(nuint code)
{
return code > unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxCode));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ZSTD_ErrorCode ERR_getErrorCode(nuint code)
{
if (!ERR_isError(code))
return 0;
return (ZSTD_ErrorCode)(0 - code);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static string ERR_getErrorName(nuint code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
/*-****************************************
* Error Strings
******************************************/
private static string ERR_getErrorString(ZSTD_ErrorCode code)
{
const string notErrorCode = "Unspecified error code";
switch (code)
{
case ZSTD_ErrorCode.ZSTD_error_no_error:
return "No error detected";
case ZSTD_ErrorCode.ZSTD_error_GENERIC:
return "Error (generic)";
case ZSTD_ErrorCode.ZSTD_error_prefix_unknown:
return "Unknown frame descriptor";
case ZSTD_ErrorCode.ZSTD_error_version_unsupported:
return "Version not supported";
case ZSTD_ErrorCode.ZSTD_error_frameParameter_unsupported:
return "Unsupported frame parameter";
case ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge:
return "Frame requires too much memory for decoding";
case ZSTD_ErrorCode.ZSTD_error_corruption_detected:
return "Data corruption detected";
case ZSTD_ErrorCode.ZSTD_error_checksum_wrong:
return "Restored data doesn't match checksum";
case ZSTD_ErrorCode.ZSTD_error_literals_headerWrong:
return "Header of Literals' block doesn't respect format specification";
case ZSTD_ErrorCode.ZSTD_error_parameter_unsupported:
return "Unsupported parameter";
case ZSTD_ErrorCode.ZSTD_error_parameter_combination_unsupported:
return "Unsupported combination of parameters";
case ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound:
return "Parameter is out of bound";
case ZSTD_ErrorCode.ZSTD_error_init_missing:
return "Context should be init first";
case ZSTD_ErrorCode.ZSTD_error_memory_allocation:
return "Allocation error : not enough memory";
case ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall:
return "workSpace buffer is not large enough";
case ZSTD_ErrorCode.ZSTD_error_stage_wrong:
return "Operation not authorized at current processing stage";
case ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge:
return "tableLog requires too much memory : unsupported";
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge:
return "Unsupported max Symbol Value : too large";
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall:
return "Specified maxSymbolValue is too small";
case ZSTD_ErrorCode.ZSTD_error_cannotProduce_uncompressedBlock:
return "This mode cannot generate an uncompressed block";
case ZSTD_ErrorCode.ZSTD_error_stabilityCondition_notRespected:
return "pledged buffer stability condition is not respected";
case ZSTD_ErrorCode.ZSTD_error_dictionary_corrupted:
return "Dictionary is corrupted";
case ZSTD_ErrorCode.ZSTD_error_dictionary_wrong:
return "Dictionary mismatch";
case ZSTD_ErrorCode.ZSTD_error_dictionaryCreation_failed:
return "Cannot create Dictionary from provided samples";
case ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall:
return "Destination buffer is too small";
case ZSTD_ErrorCode.ZSTD_error_srcSize_wrong:
return "Src size is incorrect";
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_null:
return "Operation on NULL destination buffer";
case ZSTD_ErrorCode.ZSTD_error_noForwardProgress_destFull:
return "Operation made no progress over multiple calls, due to output buffer being full";
case ZSTD_ErrorCode.ZSTD_error_noForwardProgress_inputEmpty:
return "Operation made no progress over multiple calls, due to input being empty";
case ZSTD_ErrorCode.ZSTD_error_frameIndex_tooLarge:
return "Frame index is too large";
case ZSTD_ErrorCode.ZSTD_error_seekableIO:
return "An I/O error occurred when reading/seeking";
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_wrong:
return "Destination buffer is wrong";
case ZSTD_ErrorCode.ZSTD_error_srcBuffer_wrong:
return "Source buffer is wrong";
case ZSTD_ErrorCode.ZSTD_error_sequenceProducer_failed:
return "Block-level external sequence producer returned an error code";
case ZSTD_ErrorCode.ZSTD_error_externalSequences_invalid:
return "External sequences are not valid";
case ZSTD_ErrorCode.ZSTD_error_maxCode:
default:
return notErrorCode;
}
}
}

View File

@@ -1,7 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public struct EstimatedBlockSize
{
public nuint estLitSize;
public nuint estBlockSize;
}

View File

@@ -1,19 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/*-*************************************
* Acceleration
***************************************/
public struct FASTCOVER_accel_t
{
/* Percentage of training samples used for ZDICT_finalizeDictionary */
public uint finalize;
/* Number of dmer skipped between each dmer counted in computeFrequency */
public uint skip;
public FASTCOVER_accel_t(uint finalize, uint skip)
{
this.finalize = finalize;
this.skip = skip;
}
}

View File

@@ -1,19 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/*-*************************************
* Context
***************************************/
public unsafe struct FASTCOVER_ctx_t
{
public byte* samples;
public nuint* offsets;
public nuint* samplesSizes;
public nuint nbSamples;
public nuint nbTrainSamples;
public nuint nbTestSamples;
public nuint nbDmers;
public uint* freqs;
public uint d;
public uint f;
public FASTCOVER_accel_t accelParams;
}

View File

@@ -1,12 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/**
* Parameters for FASTCOVER_tryParameters().
*/
public unsafe struct FASTCOVER_tryParameters_data_s
{
public FASTCOVER_ctx_t* ctx;
public COVER_best_s* best;
public nuint dictBufferCapacity;
public ZDICT_cover_params_t parameters;
}

View File

@@ -1,7 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public struct FPStats
{
public Fingerprint pastEvents;
public Fingerprint newEvents;
}

View File

@@ -1,16 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
Hence their body are included in next section.
*/
public unsafe struct FSE_CState_t
{
public nint value;
public void* stateTable;
public void* symbolTT;
public uint stateLog;
}

View File

@@ -1,12 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/* *****************************************
* FSE symbol decompression API
*******************************************/
public unsafe struct FSE_DState_t
{
public nuint state;
/* precise table may vary, depending on U16 */
public void* table;
}

View File

@@ -1,8 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/* ====== Decompression ====== */
public struct FSE_DTableHeader
{
public ushort tableLog;
public ushort fastMode;
}

View File

@@ -1,6 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public unsafe struct FSE_DecompressWksp
{
public fixed short ncount[256];
}

View File

@@ -1,8 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public struct FSE_decode_t
{
public ushort newState;
public byte symbol;
public byte nbBits;
}

View File

@@ -1,13 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public enum FSE_repeat
{
/**< Cannot use the previous table */
FSE_repeat_none,
/**< Can use the previous table but it must be checked */
FSE_repeat_check,
/**< Can use the previous table and it is assumed to be valid */
FSE_repeat_valid,
}

View File

@@ -1,10 +0,0 @@
namespace SharpCompress.Compressors.ZStandard.Unsafe;
/* *****************************************
* Implementation of inlined functions
*******************************************/
public struct FSE_symbolCompressionTransform
{
public int deltaFindState;
public uint deltaNbBits;
}

View File

@@ -1,761 +0,0 @@
using static SharpCompress.Compressors.ZStandard.UnsafeHelper;
namespace SharpCompress.Compressors.ZStandard.Unsafe;
public static unsafe partial class Methods
{
/*-*************************************
* Hash Functions
***************************************/
/**
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
*/
private static nuint FASTCOVER_hashPtrToIndex(void* p, uint f, uint d)
{
if (d == 6)
{
return ZSTD_hash6Ptr(p, f);
}
return ZSTD_hash8Ptr(p, f);
}
private static readonly FASTCOVER_accel_t* FASTCOVER_defaultAccelParameters = GetArrayPointer(
new FASTCOVER_accel_t[11]
{
new FASTCOVER_accel_t(finalize: 100, skip: 0),
new FASTCOVER_accel_t(finalize: 100, skip: 0),
new FASTCOVER_accel_t(finalize: 50, skip: 1),
new FASTCOVER_accel_t(finalize: 34, skip: 2),
new FASTCOVER_accel_t(finalize: 25, skip: 3),
new FASTCOVER_accel_t(finalize: 20, skip: 4),
new FASTCOVER_accel_t(finalize: 17, skip: 5),
new FASTCOVER_accel_t(finalize: 14, skip: 6),
new FASTCOVER_accel_t(finalize: 13, skip: 7),
new FASTCOVER_accel_t(finalize: 11, skip: 8),
new FASTCOVER_accel_t(finalize: 10, skip: 9),
}
);
/*-*************************************
* Helper functions
***************************************/
/**
* Selects the best segment in an epoch.
* Segments of are scored according to the function:
*
* Let F(d) be the frequency of all dmers with hash value d.
* Let S_i be hash value of the dmer at position i of segment S which has length k.
*
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
*
* Once the dmer with hash value d is in the dictionary we set F(d) = 0.
*/
private static COVER_segment_t FASTCOVER_selectSegment(
FASTCOVER_ctx_t* ctx,
uint* freqs,
uint begin,
uint end,
ZDICT_cover_params_t parameters,
ushort* segmentFreqs
)
{
/* Constants */
uint k = parameters.k;
uint d = parameters.d;
uint f = ctx->f;
uint dmersInK = k - d + 1;
/* Try each segment (activeSegment) and save the best (bestSegment) */
COVER_segment_t bestSegment = new COVER_segment_t
{
begin = 0,
end = 0,
score = 0,
};
COVER_segment_t activeSegment;
activeSegment.begin = begin;
activeSegment.end = begin;
activeSegment.score = 0;
while (activeSegment.end < end)
{
/* Get hash value of current dmer */
nuint idx = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
if (segmentFreqs[idx] == 0)
{
activeSegment.score += freqs[idx];
}
activeSegment.end += 1;
segmentFreqs[idx] += 1;
if (activeSegment.end - activeSegment.begin == dmersInK + 1)
{
/* Get hash value of the dmer to be eliminated from active segment */
nuint delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
segmentFreqs[delIndex] -= 1;
if (segmentFreqs[delIndex] == 0)
{
activeSegment.score -= freqs[delIndex];
}
activeSegment.begin += 1;
}
if (activeSegment.score > bestSegment.score)
{
bestSegment = activeSegment;
}
}
while (activeSegment.begin < end)
{
nuint delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
segmentFreqs[delIndex] -= 1;
activeSegment.begin += 1;
}
{
/* Zero the frequency of hash value of each dmer covered by the chosen segment. */
uint pos;
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos)
{
nuint i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d);
freqs[i] = 0;
}
}
return bestSegment;
}
private static int FASTCOVER_checkParameters(
ZDICT_cover_params_t parameters,
nuint maxDictSize,
uint f,
uint accel
)
{
if (parameters.d == 0 || parameters.k == 0)
{
return 0;
}
if (parameters.d != 6 && parameters.d != 8)
{
return 0;
}
if (parameters.k > maxDictSize)
{
return 0;
}
if (parameters.d > parameters.k)
{
return 0;
}
if (f > 31 || f == 0)
{
return 0;
}
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1)
{
return 0;
}
if (accel > 10 || accel == 0)
{
return 0;
}
return 1;
}
/**
* Clean up a context initialized with `FASTCOVER_ctx_init()`.
*/
private static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx)
{
if (ctx == null)
return;
free(ctx->freqs);
ctx->freqs = null;
free(ctx->offsets);
ctx->offsets = null;
}
/**
* Calculate for frequency of hash value of each dmer in ctx->samples
*/
private static void FASTCOVER_computeFrequency(uint* freqs, FASTCOVER_ctx_t* ctx)
{
uint f = ctx->f;
uint d = ctx->d;
uint skip = ctx->accelParams.skip;
uint readLength = d > 8 ? d : 8;
nuint i;
assert(ctx->nbTrainSamples >= 5);
assert(ctx->nbTrainSamples <= ctx->nbSamples);
for (i = 0; i < ctx->nbTrainSamples; i++)
{
/* start of current dmer */
nuint start = ctx->offsets[i];
nuint currSampleEnd = ctx->offsets[i + 1];
while (start + readLength <= currSampleEnd)
{
nuint dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
freqs[dmerIndex]++;
start = start + skip + 1;
}
}
}
/**
* Prepare a context for dictionary building.
* The context is only dependent on the parameter `d` and can be used multiple
* times.
* Returns 0 on success or error code on error.
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
*/
private static nuint FASTCOVER_ctx_init(
FASTCOVER_ctx_t* ctx,
void* samplesBuffer,
nuint* samplesSizes,
uint nbSamples,
uint d,
double splitPoint,
uint f,
FASTCOVER_accel_t accelParams
)
{
byte* samples = (byte*)samplesBuffer;
nuint totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
/* Split samples into testing and training sets */
uint nbTrainSamples = splitPoint < 1 ? (uint)(nbSamples * splitPoint) : nbSamples;
uint nbTestSamples = splitPoint < 1 ? nbSamples - nbTrainSamples : nbSamples;
nuint trainingSamplesSize =
splitPoint < 1 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
nuint testSamplesSize =
splitPoint < 1
? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples)
: totalSamplesSize;
if (
totalSamplesSize < (d > sizeof(ulong) ? d : sizeof(ulong))
|| totalSamplesSize >= (sizeof(nuint) == 8 ? unchecked((uint)-1) : 1 * (1U << 30))
)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
if (nbTrainSamples < 5)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
if (nbTestSamples < 1)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
*ctx = new FASTCOVER_ctx_t
{
samples = samples,
samplesSizes = samplesSizes,
nbSamples = nbSamples,
nbTrainSamples = nbTrainSamples,
nbTestSamples = nbTestSamples,
nbDmers = trainingSamplesSize - (d > sizeof(ulong) ? d : sizeof(ulong)) + 1,
d = d,
f = f,
accelParams = accelParams,
offsets = (nuint*)calloc(nbSamples + 1, (ulong)sizeof(nuint)),
};
if (ctx->offsets == null)
{
FASTCOVER_ctx_destroy(ctx);
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation));
}
{
uint i;
ctx->offsets[0] = 0;
assert(nbSamples >= 5);
for (i = 1; i <= nbSamples; ++i)
{
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
}
}
ctx->freqs = (uint*)calloc((ulong)1 << (int)f, sizeof(uint));
if (ctx->freqs == null)
{
FASTCOVER_ctx_destroy(ctx);
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation));
}
FASTCOVER_computeFrequency(ctx->freqs, ctx);
return 0;
}
/**
* Given the prepared context build the dictionary.
*/
private static nuint FASTCOVER_buildDictionary(
FASTCOVER_ctx_t* ctx,
uint* freqs,
void* dictBuffer,
nuint dictBufferCapacity,
ZDICT_cover_params_t parameters,
ushort* segmentFreqs
)
{
byte* dict = (byte*)dictBuffer;
nuint tail = dictBufferCapacity;
/* Divide the data into epochs. We will select one segment from each epoch. */
COVER_epoch_info_t epochs = COVER_computeEpochs(
(uint)dictBufferCapacity,
(uint)ctx->nbDmers,
parameters.k,
1
);
const nuint maxZeroScoreRun = 10;
nuint zeroScoreRun = 0;
nuint epoch;
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num)
{
uint epochBegin = (uint)(epoch * epochs.size);
uint epochEnd = epochBegin + epochs.size;
nuint segmentSize;
/* Select a segment */
COVER_segment_t segment = FASTCOVER_selectSegment(
ctx,
freqs,
epochBegin,
epochEnd,
parameters,
segmentFreqs
);
if (segment.score == 0)
{
if (++zeroScoreRun >= maxZeroScoreRun)
{
break;
}
continue;
}
zeroScoreRun = 0;
segmentSize =
segment.end - segment.begin + parameters.d - 1 < tail
? segment.end - segment.begin + parameters.d - 1
: tail;
if (segmentSize < parameters.d)
{
break;
}
tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, (uint)segmentSize);
}
return tail;
}
/**
* Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading.
*/
private static void FASTCOVER_tryParameters(void* opaque)
{
/* Save parameters as local variables */
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)opaque;
FASTCOVER_ctx_t* ctx = data->ctx;
ZDICT_cover_params_t parameters = data->parameters;
nuint dictBufferCapacity = data->dictBufferCapacity;
nuint totalCompressedSize = unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC));
/* Initialize array to keep track of frequency of dmer within activeSegment */
ushort* segmentFreqs = (ushort*)calloc((ulong)1 << (int)ctx->f, sizeof(ushort));
/* Allocate space for hash table, dict, and freqs */
byte* dict = (byte*)malloc(dictBufferCapacity);
COVER_dictSelection selection = COVER_dictSelectionError(
unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))
);
uint* freqs = (uint*)malloc(((ulong)1 << (int)ctx->f) * sizeof(uint));
if (segmentFreqs == null || dict == null || freqs == null)
{
goto _cleanup;
}
memcpy(freqs, ctx->freqs, (uint)(((ulong)1 << (int)ctx->f) * sizeof(uint)));
{
nuint tail = FASTCOVER_buildDictionary(
ctx,
freqs,
dict,
dictBufferCapacity,
parameters,
segmentFreqs
);
uint nbFinalizeSamples = (uint)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
selection = COVER_selectDict(
dict + tail,
dictBufferCapacity,
dictBufferCapacity - tail,
ctx->samples,
ctx->samplesSizes,
nbFinalizeSamples,
ctx->nbTrainSamples,
ctx->nbSamples,
parameters,
ctx->offsets,
totalCompressedSize
);
if (COVER_dictSelectionIsError(selection) != 0)
{
goto _cleanup;
}
}
_cleanup:
free(dict);
COVER_best_finish(data->best, parameters, selection);
free(data);
free(segmentFreqs);
COVER_dictSelectionFree(selection);
free(freqs);
}
private static void FASTCOVER_convertToCoverParams(
ZDICT_fastCover_params_t fastCoverParams,
ZDICT_cover_params_t* coverParams
)
{
coverParams->k = fastCoverParams.k;
coverParams->d = fastCoverParams.d;
coverParams->steps = fastCoverParams.steps;
coverParams->nbThreads = fastCoverParams.nbThreads;
coverParams->splitPoint = fastCoverParams.splitPoint;
coverParams->zParams = fastCoverParams.zParams;
coverParams->shrinkDict = fastCoverParams.shrinkDict;
}
private static void FASTCOVER_convertToFastCoverParams(
ZDICT_cover_params_t coverParams,
ZDICT_fastCover_params_t* fastCoverParams,
uint f,
uint accel
)
{
fastCoverParams->k = coverParams.k;
fastCoverParams->d = coverParams.d;
fastCoverParams->steps = coverParams.steps;
fastCoverParams->nbThreads = coverParams.nbThreads;
fastCoverParams->splitPoint = coverParams.splitPoint;
fastCoverParams->f = f;
fastCoverParams->accel = accel;
fastCoverParams->zParams = coverParams.zParams;
fastCoverParams->shrinkDict = coverParams.shrinkDict;
}
/*! ZDICT_trainFromBuffer_fastCover():
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
* d and k are required.
* All other parameters are optional, will use default values if not provided
* The resulting dictionary will be saved into `dictBuffer`.
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/
public static nuint ZDICT_trainFromBuffer_fastCover(
void* dictBuffer,
nuint dictBufferCapacity,
void* samplesBuffer,
nuint* samplesSizes,
uint nbSamples,
ZDICT_fastCover_params_t parameters
)
{
byte* dict = (byte*)dictBuffer;
FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
g_displayLevel = (int)parameters.zParams.notificationLevel;
parameters.splitPoint = 1;
parameters.f = parameters.f == 0 ? 20 : parameters.f;
parameters.accel = parameters.accel == 0 ? 1 : parameters.accel;
coverParams = new ZDICT_cover_params_t();
FASTCOVER_convertToCoverParams(parameters, &coverParams);
if (
FASTCOVER_checkParameters(
coverParams,
dictBufferCapacity,
parameters.f,
parameters.accel
) == 0
)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound));
}
if (nbSamples == 0)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
if (dictBufferCapacity < 256)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall));
}
accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
{
nuint initVal = FASTCOVER_ctx_init(
&ctx,
samplesBuffer,
samplesSizes,
nbSamples,
coverParams.d,
parameters.splitPoint,
parameters.f,
accelParams
);
if (ERR_isError(initVal))
{
return initVal;
}
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
{
/* Initialize array to keep track of frequency of dmer within activeSegment */
ushort* segmentFreqs = (ushort*)calloc((ulong)1 << (int)parameters.f, sizeof(ushort));
nuint tail = FASTCOVER_buildDictionary(
&ctx,
ctx.freqs,
dictBuffer,
dictBufferCapacity,
coverParams,
segmentFreqs
);
uint nbFinalizeSamples = (uint)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
nuint dictionarySize = ZDICT_finalizeDictionary(
dict,
dictBufferCapacity,
dict + tail,
dictBufferCapacity - tail,
samplesBuffer,
samplesSizes,
nbFinalizeSamples,
coverParams.zParams
);
if (!ERR_isError(dictionarySize)) { }
FASTCOVER_ctx_destroy(&ctx);
free(segmentFreqs);
return dictionarySize;
}
}
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
* The same requirements as above hold for all the parameters except `parameters`.
* This function tries many parameter combinations (specifically, k and d combinations)
* and picks the best parameters. `*parameters` is filled with the best parameters found,
* dictionary constructed with those parameters is stored in `dictBuffer`.
* All of the parameters d, k, steps, f, and accel are optional.
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
* if steps is zero it defaults to its default value.
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
* If f is zero, default value of 20 is used.
* If accel is zero, default value of 1 is used.
*
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* On success `*parameters` contains the parameters selected.
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
*/
public static nuint ZDICT_optimizeTrainFromBuffer_fastCover(
void* dictBuffer,
nuint dictBufferCapacity,
void* samplesBuffer,
nuint* samplesSizes,
uint nbSamples,
ZDICT_fastCover_params_t* parameters
)
{
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
/* constants */
uint nbThreads = parameters->nbThreads;
double splitPoint = parameters->splitPoint <= 0 ? 0.75 : parameters->splitPoint;
uint kMinD = parameters->d == 0 ? 6 : parameters->d;
uint kMaxD = parameters->d == 0 ? 8 : parameters->d;
uint kMinK = parameters->k == 0 ? 50 : parameters->k;
uint kMaxK = parameters->k == 0 ? 2000 : parameters->k;
uint kSteps = parameters->steps == 0 ? 40 : parameters->steps;
uint kStepSize = (kMaxK - kMinK) / kSteps > 1 ? (kMaxK - kMinK) / kSteps : 1;
uint kIterations = (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
uint f = parameters->f == 0 ? 20 : parameters->f;
uint accel = parameters->accel == 0 ? 1 : parameters->accel;
const uint shrinkDict = 0;
/* Local variables */
int displayLevel = (int)parameters->zParams.notificationLevel;
uint iteration = 1;
uint d;
uint k;
COVER_best_s best;
void* pool = null;
int warned = 0;
if (splitPoint <= 0 || splitPoint > 1)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound));
}
if (accel == 0 || accel > 10)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound));
}
if (kMinK < kMaxD || kMaxK < kMinK)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound));
}
if (nbSamples == 0)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong));
}
if (dictBufferCapacity < 256)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall));
}
if (nbThreads > 1)
{
pool = POOL_create(nbThreads, 1);
if (pool == null)
{
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation));
}
}
COVER_best_init(&best);
coverParams = new ZDICT_cover_params_t();
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel];
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
for (d = kMinD; d <= kMaxD; d += 2)
{
/* Initialize the context for this value of d */
FASTCOVER_ctx_t ctx;
{
nuint initVal = FASTCOVER_ctx_init(
&ctx,
samplesBuffer,
samplesSizes,
nbSamples,
d,
splitPoint,
f,
accelParams
);
if (ERR_isError(initVal))
{
COVER_best_destroy(&best);
POOL_free(pool);
return initVal;
}
}
if (warned == 0)
{
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
warned = 1;
}
for (k = kMinK; k <= kMaxK; k += kStepSize)
{
/* Prepare the arguments */
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)malloc(
(ulong)sizeof(FASTCOVER_tryParameters_data_s)
);
if (data == null)
{
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
POOL_free(pool);
return unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation));
}
data->ctx = &ctx;
data->best = &best;
data->dictBufferCapacity = dictBufferCapacity;
data->parameters = coverParams;
data->parameters.k = k;
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (uint)g_displayLevel;
if (
FASTCOVER_checkParameters(
data->parameters,
dictBufferCapacity,
data->ctx->f,
accel
) == 0
)
{
free(data);
continue;
}
COVER_best_start(&best);
if (pool != null)
{
POOL_add(
pool,
(delegate* managed<void*, void>)(&FASTCOVER_tryParameters),
data
);
}
else
{
FASTCOVER_tryParameters(data);
}
++iteration;
}
COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx);
}
{
nuint dictSize = best.dictSize;
if (ERR_isError(best.compressedSize))
{
nuint compressedSize = best.compressedSize;
COVER_best_destroy(&best);
POOL_free(pool);
return compressedSize;
}
FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
memcpy(dictBuffer, best.dict, (uint)dictSize);
COVER_best_destroy(&best);
POOL_free(pool);
return dictSize;
}
}
}

Some files were not shown because too many files have changed in this diff Show More