Compare commits

...

66 Commits

Author SHA1 Message Date
Adam Hathcock
862fa50fcb Don't use SkipLocalsInit 2021-11-29 09:29:40 +00:00
Adam Hathcock
7b87924172 Add zstd classes for initial work 2021-11-28 10:11:54 +00:00
Adam Hathcock
d9d7ea8ec5 Add zstandard compression 2021-11-28 09:52:09 +00:00
Adam Hathcock
8d17d09455 Merge pull request #624 from adamhathcock/issue-617
Add test and probable fix for Issue 617
2021-11-22 09:20:15 +00:00
Adam Hathcock
05208ccd9b Add test and probable fix for Issue 617 2021-11-22 08:40:40 +00:00
Adam Hathcock
a1e7c0068d Merge pull request #622 from adamhathcock/net461-tests
Net461 tests
2021-10-02 15:32:20 +01:00
Adam Hathcock
e6bec19946 Mark for 0.30 2021-10-02 15:29:22 +01:00
Adam Hathcock
ec2be2869f Fix whitespace from dotnet format 2021-10-02 15:29:03 +01:00
Adam Hathcock
ce5432ed73 Fix tests for multi-targetting 2021-10-02 15:25:43 +01:00
Adam Hathcock
b6e0ad89ce Remove duplicated artifact step 2021-10-02 15:21:05 +01:00
Adam Hathcock
2745bfa19b Minor SDK update 2021-10-02 15:19:51 +01:00
Adam Hathcock
3cdc4b38a6 Test 461 on github actions 2021-10-02 15:19:13 +01:00
Adam Hathcock
fc1ca808d7 Merge pull request #621 from inthemedium/master
Add net461 target to clean up issues with system.* nuget dependencies
2021-10-02 15:08:18 +01:00
Jeff Tyson
6983e66037 Fix preprocessor condition 2021-10-01 16:34:00 +00:00
Jeff Tyson
01f7336d09 Based on docs, the target should bet net461 2021-09-29 22:04:47 +00:00
Jeff Tyson
1561bba538 Add net462 target to clean up issues with system.* nuget dependencies 2021-09-29 21:55:11 +00:00
Adam Hathcock
3ecf8a5e0c Merge pull request #616 from amosonn/patch-1
Fix for chunked read for ZLibBaseStream
2021-09-27 09:14:58 +01:00
Adam Hathcock
e2095fc416 Merge branch 'master' into patch-1 2021-09-27 09:08:58 +01:00
Amos Onn
8398d40106 Fix #615 2021-09-14 22:02:18 +02:00
Amos Onn
134fa8892f Test for bug #615 2021-09-14 21:55:05 +02:00
Adam Hathcock
ea5c8dc063 Merge pull request #614 from adamhathcock/ensure-dest-dir-exists
Ensure destination directory exists.
2021-09-12 08:56:42 +01:00
Adam Hathcock
0209d00164 Minor updates and prep for 0.29 2021-09-12 08:52:00 +01:00
Adam Hathcock
a8d065dc9e Ensure destination directory exists 2021-09-12 08:47:30 +01:00
Adam Hathcock
7bd9711ade Merge pull request #610 from cyr/master
Bugfix for TarWriter - too much padding in large files
2021-09-12 08:43:20 +01:00
cyr
61802eadb4 Merge branch 'adamhathcock:master' into master 2021-09-12 09:37:07 +02:00
Adam Hathcock
b425659058 Merge pull request #611 from Thunderstr1k3/fix-zipheader-seeking
Allowing to seek empty zip files
2021-09-12 08:28:05 +01:00
Christian
3e32e3d7b1 Allowing to seek empty zip files 2021-09-02 13:54:32 +02:00
cyr
1b661c9df1 Fixed bug where large (int32+ file size) adds an additional 512 bytes of padding in tar files. 2021-08-27 22:38:04 +02:00
Adam Hathcock
54fc26b93d Update build and mark for 0.28.3 2021-06-04 13:43:35 +01:00
Adam Hathcock
161f99bbad Merge pull request #601 from salvois/master
Write ZIP64 End of Central Directory only if needed.
2021-06-04 13:22:01 +01:00
Adam Hathcock
c012db0776 Merge branch 'master' into master 2021-06-04 13:17:13 +01:00
Adam Hathcock
8ee257d299 Merge pull request #592 from adamhathcock/memory-downgrade
Downgrade System.Memory to fix buffer version issue
2021-06-04 13:16:40 +01:00
Adam Hathcock
f9522107c3 Merge branch 'master' into memory-downgrade 2021-06-04 13:16:34 +01:00
Adam Hathcock
e07046a37a Merge pull request #596 from DannyBoyk/issue_595_conditionally_read_zip64_extra
Conditionally parse Zip64 extra field based on specification
2021-06-04 13:07:08 +01:00
Salvatore Isaja
ad6d0d9ae8 Write ZIP64 End of Central Directory only if needed. 2021-05-23 21:10:35 +02:00
Daniel Nash
fdc33e91bd Conditionally parse Zip64 extra field based on specification
The Zip64 extra field should look for values based on the corresponding
values in the local entry header.

Fixes adamhathcock/sharpcompress#595
2021-04-26 14:58:10 -04:00
Adam Hathcock
a34f5a855c Mark for 0.28.2 2021-04-25 09:29:56 +01:00
Adam Hathcock
6474741af1 Merge pull request #593 from adamhathcock/fix-pkware-encryption
ReadFully used by pkware encryption didn’t like spans
2021-04-25 09:29:02 +01:00
Adam Hathcock
c10bd840c5 ReadFully used by pkware encryption didn’t like spans 2021-04-25 09:25:51 +01:00
Adam Hathcock
e6dded826b Downgrade System.Memory to fix buffer version issue 2021-04-24 09:16:46 +01:00
Adam Hathcock
8a022c4b18 Update FORMATS.md
remove LZipArchive/Reader/Writer mention
2021-03-28 08:58:11 +01:00
Adam Hathcock
cfef228afc Merge pull request #579 from Looooong/fix/do-not-place-extention-classes-in-common-namespace
Do not place extension classes in common namespace
2021-03-18 13:52:40 +00:00
Nguyễn Đức Long
237ff9f055 Do not place extension classes in common namespace 2021-03-18 20:44:04 +07:00
Adam Hathcock
020f862814 Bug fix for recursive call introduced in 0.28 2021-02-18 08:31:50 +00:00
Adam Hathcock
fa6107200d Merge pull request #572 from Erior/feature/521
Not so elegant perhaps for checking 7z encryption
2021-02-16 08:05:08 +00:00
Adam Hathcock
eb81f972c4 Merge branch 'master' into feature/521 2021-02-16 08:01:32 +00:00
Lars Vahlenberg
93c1ff396e Not so elegant perhaps 2021-02-14 16:29:01 +01:00
Adam Hathcock
403baf05a6 Mark for 0.28 2021-02-14 13:07:35 +00:00
Adam Hathcock
a51b56339a Fix complete entry check for RAR files. 2021-02-14 13:00:43 +00:00
Adam Hathcock
f48a6d47dc Merge pull request #571 from Erior/feature/540
Proposal fixing Extra bytes written when setting zip64
2021-02-14 12:54:17 +00:00
Adam Hathcock
5b52463e4c Merge pull request #570 from Erior/feature/555
Propsal for handling Zip with long comment
2021-02-14 12:52:42 +00:00
Adam Hathcock
6f08bb72d8 Merge pull request #569 from BrendanGrant/improve_how_missing_parts_are_handled
Improve how missing parts are handled
2021-02-14 12:49:49 +00:00
Lars Vahlenberg
045093f453 Linux is case sensitive with files names 2021-02-14 10:26:26 +01:00
Lars Vahlenberg
566c49ce53 Proposal
Zip64 requires version 4.5
Number of disks is 4 bytes and not 8
2021-02-14 02:42:32 +01:00
Lars Vahlenberg
d1d2758ee0 Propsal for handling Zip with long comment 2021-02-13 23:57:03 +01:00
Brendan Grant
5b86c40d5b Properly detect if RAR is complete at the end or not 2021-02-13 13:34:57 -06:00
Brendan Grant
53393e744e Supporting reading contents of incomplete files 2021-02-13 13:33:43 -06:00
Adam Hathcock
2dd17e3882 Be explicit about zip64 extra field sizes. Formatting 2021-02-13 07:05:53 +00:00
Adam Hathcock
c4f7433584 Merge pull request #567 from Nanook/master
Zip64 Header and Size fix
2021-02-13 06:58:41 +00:00
Adam Hathcock
9405a7cf4b Merge pull request #568 from Bond-009/stackalloc
Use stackallocs where possible/sensible
2021-02-13 06:39:32 +00:00
Bond_009
cd677440ce Use stackallocs where possible/sensible 2021-02-12 20:20:15 +01:00
Craig Greenhill
c06f4bc5a8 Zip64 Header and Size fix 2021-02-11 09:37:59 +00:00
Adam Hathcock
4a7337b223 Merge pull request #563 from adamhathcock/add-reader-test-gzip
Fix Rewindable stream Length and add GZip Reader tests
2021-01-13 15:13:34 +00:00
Adam Hathcock
1d8afb817e Bump version 2021-01-13 14:41:25 +00:00
Adam Hathcock
0f06c3d934 Fix rewindable stream to expose length 2021-01-13 14:40:36 +00:00
Adam Hathcock
9d5cb8d119 Add GZip Reader tests 2021-01-13 10:42:59 +00:00
267 changed files with 42214 additions and 256 deletions

View File

@@ -12,13 +12,9 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-dotnet@v1
with:
dotnet-version: 5.0.101
dotnet-version: 5.0.401
- run: dotnet run -p build/build.csproj
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.os }}-sharpcompress.nupkg
path: artifacts/*
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.os }}-sharpcompress.snupkg
path: artifacts/*

View File

@@ -19,7 +19,6 @@
| Tar.XZ | LZMA2 | Decompress | TarArchive | TarReader | TarWriter (3) |
| GZip (single file) | DEFLATE | Both | GZipArchive | GZipReader | GZipWriter |
| 7Zip (4) | LZMA, LZMA2, BZip2, PPMd, BCJ, BCJ2, Deflate | Decompress | SevenZipArchive | N/A | N/A |
| LZip (single file) (5) | LZip (LZMA) | Both | LZipArchive | LZipReader | LZipWriter |
1. SOLID Rars are only supported in the RarReader API.
2. Zip format supports pkware and WinzipAES encryption. However, encrypted LZMA is not supported. Zip64 reading/writing is supported but only with seekable streams as the Zip spec doesn't support Zip64 data in post data descriptors. Deflate64 is only supported for reading.

View File

@@ -49,20 +49,20 @@ class Program
Target(Build, DependsOn(Format),
framework =>
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && framework == "net46")
{
return;
}
Run("dotnet", "build src/SharpCompress/SharpCompress.csproj -c Release");
});
Target(Test, DependsOn(Build), ForEach("net5.0"),
Target(Test, DependsOn(Build), ForEach("net5.0", "net461"),
framework =>
{
IEnumerable<string> GetFiles(string d)
{
return Glob.Files(".", d);
}
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && framework == "net461")
{
return;
}
foreach (var file in GetFiles("**/*.Test.csproj"))
{

View File

@@ -6,9 +6,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Bullseye" Version="3.6.0" />
<PackageReference Include="Bullseye" Version="3.8.0" />
<PackageReference Include="Glob" Version="1.1.8" />
<PackageReference Include="SimpleExec" Version="6.4.0" />
<PackageReference Include="SimpleExec" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -1,5 +1,6 @@
{
"sdk": {
"version": "5.0.101"
"version": "5.0.300",
"rollForward": "latestFeature"
}
}

View File

@@ -4,7 +4,7 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
#if !NETSTANDARD2_0 && !NETSTANDARD2_1
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
#endif
@@ -22,7 +22,7 @@ namespace SharpCompress.Algorithms
/// </summary>
public const uint SeedValue = 1U;
#if !NETSTANDARD2_0 && !NETSTANDARD2_1
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
private const int MinBufferSize = 64;
#endif
@@ -51,12 +51,7 @@ namespace SharpCompress.Algorithms
/// <returns>The <see cref="uint"/>.</returns>
public static uint Calculate(uint adler, ReadOnlySpan<byte> buffer)
{
if (buffer.IsEmpty)
{
return SeedValue;
}
#if !NETSTANDARD2_0 && !NETSTANDARD2_1
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
if (Sse3.IsSupported && buffer.Length >= MinBufferSize)
{
return CalculateSse(adler, buffer);
@@ -69,7 +64,7 @@ namespace SharpCompress.Algorithms
}
// Based on https://github.com/chromium/chromium/blob/master/third_party/zlib/adler32_simd.c
#if !NETSTANDARD2_0 && !NETSTANDARD2_1
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
private static unsafe uint CalculateSse(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
@@ -282,4 +277,4 @@ namespace SharpCompress.Algorithms
return (s2 << 16) | s1;
}
}
}
}

View File

@@ -98,7 +98,7 @@ namespace SharpCompress.Archives.GZip
public static bool IsGZipFile(Stream stream)
{
// read the header on the first read
byte[] header = new byte[10];
Span<byte> header = stackalloc byte[10];
// workitem 8501: handle edge case (decompress empty stream)
if (!stream.ReadFully(header))

View File

@@ -10,7 +10,8 @@ using SharpCompress.Readers.Rar;
namespace SharpCompress.Archives.Rar
{
public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
public class
RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
{
internal Lazy<IRarUnpack> UnpackV2017 { get; } = new Lazy<IRarUnpack>(() => new SharpCompress.Compressors.Rar.UnpackV2017.Unpack());
internal Lazy<IRarUnpack> UnpackV1 { get; } = new Lazy<IRarUnpack>(() => new SharpCompress.Compressors.Rar.UnpackV1.Unpack());
@@ -42,7 +43,7 @@ namespace SharpCompress.Archives.Rar
protected override IEnumerable<RarArchiveEntry> LoadEntries(IEnumerable<RarVolume> volumes)
{
return RarArchiveEntryFactory.GetEntries(this, volumes);
return RarArchiveEntryFactory.GetEntries(this, volumes, ReaderOptions);
}
protected override IEnumerable<RarVolume> LoadVolumes(IEnumerable<Stream> streams)

View File

@@ -6,6 +6,7 @@ using SharpCompress.Common;
using SharpCompress.Common.Rar;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.Compressors.Rar;
using SharpCompress.Readers;
namespace SharpCompress.Archives.Rar
{
@@ -13,11 +14,13 @@ namespace SharpCompress.Archives.Rar
{
private readonly ICollection<RarFilePart> parts;
private readonly RarArchive archive;
private readonly ReaderOptions readerOptions;
internal RarArchiveEntry(RarArchive archive, IEnumerable<RarFilePart> parts)
internal RarArchiveEntry(RarArchive archive, IEnumerable<RarFilePart> parts, ReaderOptions readerOptions)
{
this.parts = parts.ToList();
this.archive = archive;
this.readerOptions = readerOptions;
}
public override CompressionType CompressionType => CompressionType.Rar;
@@ -69,13 +72,14 @@ namespace SharpCompress.Archives.Rar
{
get
{
return parts.Select(fp => fp.FileHeader).Any(fh => !fh.IsSplitAfter);
var headers = parts.Select(x => x.FileHeader);
return !headers.First().IsSplitBefore && !headers.Last().IsSplitAfter;
}
}
private void CheckIncomplete()
{
if (!IsComplete)
if (!readerOptions.DisableCheckIncomplete && !IsComplete)
{
throw new IncompleteArchiveException("ArchiveEntry is incomplete and cannot perform this operation.");
}

View File

@@ -1,5 +1,6 @@
using System.Collections.Generic;
using SharpCompress.Common.Rar;
using SharpCompress.Readers;
namespace SharpCompress.Archives.Rar
{
@@ -36,11 +37,12 @@ namespace SharpCompress.Archives.Rar
}
internal static IEnumerable<RarArchiveEntry> GetEntries(RarArchive archive,
IEnumerable<RarVolume> rarParts)
IEnumerable<RarVolume> rarParts,
ReaderOptions readerOptions)
{
foreach (var groupedParts in GetMatchedFileParts(rarParts))
{
yield return new RarArchiveEntry(archive, groupedParts);
yield return new RarArchiveEntry(archive, groupedParts, readerOptions);
}
}
}

View File

@@ -5,7 +5,7 @@ using System.Runtime.CompilerServices;
[assembly: AssemblyTitle("SharpCompress")]
[assembly: AssemblyProduct("SharpCompress")]
[assembly: InternalsVisibleTo("SharpCompress.Test" + SharpCompress.AssemblyInfo.PublicKeySuffix)]
[assembly: CLSCompliant(true)]
[assembly: CLSCompliant(false)]
namespace SharpCompress
{

View File

@@ -36,7 +36,7 @@ namespace SharpCompress.Common
Password = password;
}
#if !NET461
#if !NETFRAMEWORK
static ArchiveEncoding()
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

View File

@@ -6,6 +6,7 @@
Zip,
Tar,
SevenZip,
GZip
GZip,
ZStandard
}
}

View File

@@ -14,6 +14,7 @@
LZip,
Xz,
Unknown,
Deflate64
Deflate64,
ZStandard,
}
}

View File

@@ -14,14 +14,25 @@ namespace SharpCompress.Common
Action<string, ExtractionOptions?> write)
{
string destinationFileName;
string file = Path.GetFileName(entry.Key);
string fullDestinationDirectoryPath = Path.GetFullPath(destinationDirectory);
//check for trailing slash.
if (fullDestinationDirectoryPath[fullDestinationDirectoryPath.Length - 1] != Path.DirectorySeparatorChar)
{
fullDestinationDirectoryPath += Path.DirectorySeparatorChar;
}
if (!Directory.Exists(fullDestinationDirectoryPath))
{
throw new ExtractionException($"Directory does not exist to extract to: {fullDestinationDirectoryPath}");
}
options ??= new ExtractionOptions()
{
Overwrite = true
};
string file = Path.GetFileName(entry.Key);
if (options.ExtractFullPath)
{
string folder = Path.GetDirectoryName(entry.Key)!;

View File

@@ -110,13 +110,13 @@ namespace SharpCompress.Common.GZip
private string ReadZeroTerminatedString(Stream stream)
{
byte[] buf1 = new byte[1];
Span<byte> buf1 = stackalloc byte[1];
var list = new List<byte>();
bool done = false;
do
{
// workitem 7740
int n = stream.Read(buf1, 0, 1);
int n = stream.Read(buf1);
if (n != 1)
{
throw new ZlibException("Unexpected EOF reading GZIP header.");

View File

@@ -437,6 +437,7 @@ namespace SharpCompress.Common.Rar.Headers
internal long DataStartPosition { get; set; }
public Stream PackedStream { get; set; }
public bool IsSplitBefore => IsRar5 ? HasHeaderFlag(HeaderFlagsV5.SPLIT_BEFORE) : HasFlag(FileFlagsV4.SPLIT_BEFORE);
public bool IsSplitAfter => IsRar5 ? HasHeaderFlag(HeaderFlagsV5.SPLIT_AFTER) : HasFlag(FileFlagsV4.SPLIT_AFTER);
public bool IsDirectory => HasFlag(IsRar5 ? FileFlagsV5.DIRECTORY : FileFlagsV4.DIRECTORY);

View File

@@ -50,11 +50,11 @@ namespace SharpCompress.Common.Rar
if (sizeToRead > 0)
{
int alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf);
byte[] cipherText = new byte[RarRijndael.CRYPTO_BLOCK_SIZE];
Span<byte> cipherText = stackalloc byte[RarRijndael.CRYPTO_BLOCK_SIZE];
for (int i = 0; i < alignedSize / 16; i++)
{
//long ax = System.currentTimeMillis();
_actualStream.Read(cipherText, 0, RarRijndael.CRYPTO_BLOCK_SIZE);
_actualStream.Read(cipherText);
var readBytes = _rijndael.ProcessBlock(cipherText);
foreach (var readByte in readBytes)

View File

@@ -1517,6 +1517,7 @@ namespace SharpCompress.Common.SevenZip
}
}
byte[] buffer = null;
foreach (CExtractFolderInfo efi in extractFolderInfoVector)
{
int startIndex;
@@ -1553,7 +1554,7 @@ namespace SharpCompress.Common.SevenZip
Stream s = DecoderStreamHelper.CreateDecoderStream(_stream, folderStartPackPos, packSizes,
folderInfo, db.PasswordProvider);
byte[] buffer = new byte[4 << 10];
buffer ??= new byte[4 << 10];
for (; ; )
{
int processed = s.Read(buffer, 0, buffer.Length);

View File

@@ -32,7 +32,7 @@ namespace SharpCompress.Common.SevenZip
public override DateTime? ArchivedTime => null;
public override bool IsEncrypted => false;
public override bool IsEncrypted => FilePart.IsEncrypted;
public override bool IsDirectory => FilePart.Header.IsDir;

View File

@@ -102,5 +102,7 @@ namespace SharpCompress.Common.SevenZip
throw new NotImplementedException();
}
}
internal bool IsEncrypted => Folder!._coders.FindIndex(c => c._methodId._id == CMethodId.K_AES_ID) != -1;
}
}

View File

@@ -97,7 +97,7 @@ namespace SharpCompress.Common.Tar.Headers
{
numPaddingBytes = BLOCK_SIZE;
}
output.Write(new byte[numPaddingBytes], 0, numPaddingBytes);
output.Write(stackalloc byte[numPaddingBytes]);
}
internal bool Read(BinaryReader reader)

View File

@@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Common.GZip;
namespace SharpCompress.Common.ZStandard
{
public class ZStandardEntry : Entry
{
private readonly ZStandardFilePart _filePart;
internal ZStandardEntry(ZStandardFilePart filePart)
{
_filePart = filePart;
}
public override CompressionType CompressionType => CompressionType.GZip;
public override long Crc => _filePart.Crc ?? 0;
public override string Key => _filePart.FilePartName;
public override string? LinkTarget => null;
public override long CompressedSize => 0;
public override long Size => _filePart.UncompressedSize ?? 0;
public override DateTime? LastModifiedTime => _filePart.DateModified;
public override DateTime? CreatedTime => null;
public override DateTime? LastAccessedTime => null;
public override DateTime? ArchivedTime => null;
public override bool IsEncrypted => false;
public override bool IsDirectory => false;
public override bool IsSplitAfter => false;
internal override IEnumerable<FilePart> Parts => _filePart.AsEnumerable<FilePart>();
internal static IEnumerable<GZipEntry> GetEntries(Stream stream, OptionsBase options)
{
yield return new GZipEntry(new GZipFilePart(stream, options.ArchiveEncoding));
}
}
}

View File

@@ -0,0 +1,37 @@
using System;
using System.IO;
using ZstdSharp;
namespace SharpCompress.Common.ZStandard
{
internal sealed class ZStandardFilePart : FilePart
{
private string _name = "";
private readonly Stream _stream;
internal ZStandardFilePart(Stream stream, ArchiveEncoding archiveEncoding)
: base(archiveEncoding)
{
_stream = stream;
EntryStartPosition = stream.Position;
}
internal long EntryStartPosition { get; }
internal DateTime? DateModified { get; private set; }
internal int? Crc { get; private set; }
internal int? UncompressedSize { get; private set; }
internal override string FilePartName => _name!;
internal override Stream GetCompressedStream()
{
return new DecompressionStream(_stream);
}
internal override Stream GetRawStream()
{
return _stream;
}
}
}

View File

@@ -0,0 +1,23 @@
using System.IO;
using SharpCompress.Readers;
namespace SharpCompress.Common.ZStandard
{
public class ZStandardVolume : Volume
{
public ZStandardVolume(Stream stream, ReaderOptions options)
: base(stream, options)
{
}
public ZStandardVolume(FileInfo fileInfo, ReaderOptions options)
: base(fileInfo.OpenRead(), options)
{
options.LeaveStreamOpen = false;
}
public override bool IsFirstVolume => true;
public override bool IsMultiVolume => true;
}
}

View File

@@ -63,6 +63,8 @@ namespace SharpCompress.Common.Zip.Headers
var zip64ExtraData = Extra.OfType<Zip64ExtendedInformationExtraField>().FirstOrDefault();
if (zip64ExtraData != null)
{
zip64ExtraData.Process(UncompressedSize, CompressedSize, RelativeOffsetOfEntryHeader, DiskNumberStart);
if (CompressedSize == uint.MaxValue)
{
CompressedSize = zip64ExtraData.CompressedSize;

View File

@@ -53,6 +53,8 @@ namespace SharpCompress.Common.Zip.Headers
var zip64ExtraData = Extra.OfType<Zip64ExtendedInformationExtraField>().FirstOrDefault();
if (zip64ExtraData != null)
{
zip64ExtraData.Process(UncompressedSize, CompressedSize, 0, 0);
if (CompressedSize == uint.MaxValue)
{
CompressedSize = zip64ExtraData.CompressedSize;

View File

@@ -66,62 +66,74 @@ namespace SharpCompress.Common.Zip.Headers
public Zip64ExtendedInformationExtraField(ExtraDataType type, ushort length, byte[] dataBytes)
: base(type, length, dataBytes)
{
Process();
}
//From the spec values are only in the extradata if the standard
//value is set to 0xFFFF, but if one of the sizes are present, both are.
//Hence if length == 4 volume only
// if length == 8 offset only
// if length == 12 offset + volume
// if length == 16 sizes only
// if length == 20 sizes + volume
// if length == 24 sizes + offset
// if length == 28 everything.
//It is unclear how many of these are used in the wild.
private void Process()
// From the spec, values are only in the extradata if the standard
// value is set to 0xFFFFFFFF (or 0xFFFF for the Disk Start Number).
// Values, if present, must appear in the following order:
// - Original Size
// - Compressed Size
// - Relative Header Offset
// - Disk Start Number
public void Process(long uncompressedFileSize, long compressedFileSize, long relativeHeaderOffset, ushort diskNumber)
{
switch (DataBytes.Length)
var bytesRequired = ((uncompressedFileSize == uint.MaxValue) ? 8 : 0)
+ ((compressedFileSize == uint.MaxValue) ? 8 : 0)
+ ((relativeHeaderOffset == uint.MaxValue) ? 8 : 0)
+ ((diskNumber == ushort.MaxValue) ? 4 : 0);
var currentIndex = 0;
if (bytesRequired > DataBytes.Length)
{
case 4:
VolumeNumber = BinaryPrimitives.ReadUInt32LittleEndian(DataBytes);
return;
case 8:
RelativeOffsetOfEntryHeader = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
return;
case 12:
RelativeOffsetOfEntryHeader = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
VolumeNumber = BinaryPrimitives.ReadUInt32LittleEndian(DataBytes.AsSpan(8));
return;
case 16:
UncompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
CompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(8));
return;
case 20:
UncompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
CompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(8));
VolumeNumber = BinaryPrimitives.ReadUInt32LittleEndian(DataBytes.AsSpan(16));
return;
case 24:
UncompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
CompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(8));
RelativeOffsetOfEntryHeader = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(16));
return;
case 28:
UncompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes);
CompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(8));
RelativeOffsetOfEntryHeader = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(16));
VolumeNumber = BinaryPrimitives.ReadUInt32LittleEndian(DataBytes.AsSpan(24));
return;
default:
throw new ArchiveException("Unexpected size of of Zip64 extended information extra field");
throw new ArchiveException("Zip64 extended information extra field is not large enough for the required information");
}
if (uncompressedFileSize == uint.MaxValue)
{
UncompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(currentIndex));
currentIndex += 8;
}
if (compressedFileSize == uint.MaxValue)
{
CompressedSize = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(currentIndex));
currentIndex += 8;
}
if (relativeHeaderOffset == uint.MaxValue)
{
RelativeOffsetOfEntryHeader = BinaryPrimitives.ReadInt64LittleEndian(DataBytes.AsSpan(currentIndex));
currentIndex += 8;
}
if (diskNumber == ushort.MaxValue)
{
VolumeNumber = BinaryPrimitives.ReadUInt32LittleEndian(DataBytes.AsSpan(currentIndex));
}
}
/// <summary>
/// Uncompressed file size. Only valid after <see cref="Process(long, long, long, ushort)"/> has been called and if the
/// original entry header had a corresponding 0xFFFFFFFF value.
/// </summary>
public long UncompressedSize { get; private set; }
/// <summary>
/// Compressed file size. Only valid after <see cref="Process(long, long, long, ushort)"/> has been called and if the
/// original entry header had a corresponding 0xFFFFFFFF value.
/// </summary>
public long CompressedSize { get; private set; }
/// <summary>
/// Relative offset of the entry header. Only valid after <see cref="Process(long, long, long, ushort)"/> has been called and if the
/// original entry header had a corresponding 0xFFFFFFFF value.
/// </summary>
public long RelativeOffsetOfEntryHeader { get; private set; }
/// <summary>
/// Volume number. Only valid after <see cref="Process(long, long, long, ushort)"/> has been called and if the
/// original entry header had a corresponding 0xFFFF value.
/// </summary>
public uint VolumeNumber { get; private set; }
}

View File

@@ -105,6 +105,6 @@ namespace SharpCompress.Common.Zip.Headers
internal ZipFilePart Part { get; set; }
internal bool IsZip64 => CompressedSize == uint.MaxValue;
internal bool IsZip64 => CompressedSize >= uint.MaxValue;
}
}

View File

@@ -8,7 +8,10 @@ namespace SharpCompress.Common.Zip
{
internal sealed class SeekableZipHeaderFactory : ZipHeaderFactory
{
private const int MAX_ITERATIONS_FOR_DIRECTORY_HEADER = 4096;
private const int MINIMUM_EOCD_LENGTH = 22;
private const int ZIP64_EOCD_LENGTH = 20;
// Comment may be within 64kb + structure 22 bytes
private const int MAX_SEARCH_LENGTH_FOR_EOCD = 65557;
private bool _zip64;
internal SeekableZipHeaderFactory(string? password, ArchiveEncoding archiveEncoding)
@@ -20,14 +23,24 @@ namespace SharpCompress.Common.Zip
{
var reader = new BinaryReader(stream);
SeekBackToHeader(stream, reader, DIRECTORY_END_HEADER_BYTES);
SeekBackToHeader(stream, reader);
var eocd_location = stream.Position;
var entry = new DirectoryEndHeader();
entry.Read(reader);
if (entry.IsZip64)
{
_zip64 = true;
SeekBackToHeader(stream, reader, ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR);
// ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR should be before the EOCD
stream.Seek(eocd_location - ZIP64_EOCD_LENGTH - 4, SeekOrigin.Begin);
uint zip64_locator = reader.ReadUInt32();
if (zip64_locator != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR)
{
throw new ArchiveException("Failed to locate the Zip64 Directory Locator");
}
var zip64Locator = new Zip64DirectoryEndLocatorHeader();
zip64Locator.Read(reader);
@@ -73,27 +86,50 @@ namespace SharpCompress.Common.Zip
}
}
private static void SeekBackToHeader(Stream stream, BinaryReader reader, uint headerSignature)
private static bool IsMatch(byte[] haystack, int position, byte[] needle)
{
long offset = 0;
uint signature;
int iterationCount = 0;
do
for (int i = 0; i < needle.Length; i++)
{
if ((stream.Length + offset) - 4 < 0)
if (haystack[position + i] != needle[i])
{
throw new ArchiveException("Failed to locate the Zip Header");
}
stream.Seek(offset - 4, SeekOrigin.End);
signature = reader.ReadUInt32();
offset--;
iterationCount++;
if (iterationCount > MAX_ITERATIONS_FOR_DIRECTORY_HEADER)
{
throw new ArchiveException("Could not find Zip file Directory at the end of the file. File may be corrupted.");
return false;
}
}
while (signature != headerSignature);
return true;
}
private static void SeekBackToHeader(Stream stream, BinaryReader reader)
{
// Minimum EOCD length
if (stream.Length < MINIMUM_EOCD_LENGTH)
{
throw new ArchiveException("Could not find Zip file Directory at the end of the file. File may be corrupted.");
}
int len = stream.Length < MAX_SEARCH_LENGTH_FOR_EOCD ? (int)stream.Length : MAX_SEARCH_LENGTH_FOR_EOCD;
// We search for marker in reverse to find the first occurance
byte[] needle = { 0x06, 0x05, 0x4b, 0x50 };
stream.Seek(-len, SeekOrigin.End);
byte[] seek = reader.ReadBytes(len);
// Search in reverse
Array.Reverse(seek);
// don't exclude the minimum eocd region, otherwise you fail to locate the header in empty zip files
var max_search_area = len; // - MINIMUM_EOCD_LENGTH;
for (int pos_from_end = 0; pos_from_end < max_search_area; ++pos_from_end)
{
if (IsMatch(seek, pos_from_end, needle))
{
stream.Seek(-pos_from_end, SeekOrigin.End);
return;
}
}
throw new ArchiveException("Failed to locate the Zip Header");
}
internal LocalEntryHeader GetLocalHeader(Stream stream, DirectoryEntryHeader directoryEntryHeader)

View File

@@ -75,7 +75,7 @@ namespace SharpCompress.Common.Zip
if (disposing)
{
//read out last 10 auth bytes
var ten = new byte[10];
Span<byte> ten = stackalloc byte[10];
_stream.ReadFully(ten);
_stream.Dispose();
}

View File

@@ -93,7 +93,7 @@ namespace SharpCompress.Common.Zip
}
case ZipCompressionMethod.PPMd:
{
var props = new byte[2];
Span<byte> props = stackalloc byte[2];
stream.ReadFully(props);
return new PpmdStream(new PpmdProperties(props), stream, false);
}

View File

@@ -83,7 +83,7 @@ namespace SharpCompress.Compressors.BZip2
stream.SetLength(value);
}
#if !NET461 && !NETSTANDARD2_0
#if !NETFRAMEWORK && !NETSTANDARD2_0
public override int Read(Span<byte> buffer)
{
@@ -123,4 +123,4 @@ namespace SharpCompress.Compressors.BZip2
return true;
}
}
}
}

View File

@@ -256,17 +256,15 @@ namespace SharpCompress.Compressors.Deflate
}
// Read and potentially verify the GZIP trailer: CRC32 and size mod 2^32
byte[] trailer = new byte[8];
Span<byte> trailer = stackalloc byte[8];
// workitem 8679
if (_z.AvailableBytesIn != 8)
{
// Make sure we have read to the end of the stream
Array.Copy(_z.InputBuffer, _z.NextIn, trailer, 0, _z.AvailableBytesIn);
_z.InputBuffer.AsSpan(_z.NextIn, _z.AvailableBytesIn).CopyTo(trailer);
int bytesNeeded = 8 - _z.AvailableBytesIn;
int bytesRead = _stream.Read(trailer,
_z.AvailableBytesIn,
bytesNeeded);
int bytesRead = _stream.Read(trailer.Slice(_z.AvailableBytesIn, bytesNeeded));
if (bytesNeeded != bytesRead)
{
throw new ZlibException(String.Format(
@@ -276,12 +274,12 @@ namespace SharpCompress.Compressors.Deflate
}
else
{
Array.Copy(_z.InputBuffer, _z.NextIn, trailer, 0, trailer.Length);
_z.InputBuffer.AsSpan(_z.NextIn, trailer.Length).CopyTo(trailer);
}
Int32 crc32_expected = BinaryPrimitives.ReadInt32LittleEndian(trailer);
Int32 crc32_actual = crc.Crc32Result;
Int32 isize_expected = BinaryPrimitives.ReadInt32LittleEndian(trailer.AsSpan(4));
Int32 isize_expected = BinaryPrimitives.ReadInt32LittleEndian(trailer.Slice(4));
Int32 isize_actual = (Int32)(_z.TotalBytesOut & 0x00000000FFFFFFFF);
if (crc32_actual != crc32_expected)
@@ -504,13 +502,37 @@ namespace SharpCompress.Compressors.Deflate
throw new ZlibException("Cannot Read after Writing.");
}
int rc = 0;
// set up the output of the deflate/inflate codec:
_z.OutputBuffer = buffer;
_z.NextOut = offset;
_z.AvailableBytesOut = count;
if (count == 0)
{
return 0;
}
if (nomoreinput && _wantCompress)
{
return 0; // workitem 8557
// no more input data available; therefore we flush to
// try to complete the read
rc = _z.Deflate(FlushType.Finish);
if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END)
{
throw new ZlibException(String.Format("Deflating: rc={0} msg={1}", rc, _z.Message));
}
rc = (count - _z.AvailableBytesOut);
// calculate CRC after reading
if (crc != null)
{
crc.SlurpBlock(buffer, offset, rc);
}
return rc;
}
if (buffer is null)
{
@@ -529,13 +551,6 @@ namespace SharpCompress.Compressors.Deflate
throw new ArgumentOutOfRangeException(nameof(count));
}
int rc = 0;
// set up the output of the deflate/inflate codec:
_z.OutputBuffer = buffer;
_z.NextOut = offset;
_z.AvailableBytesOut = count;
// This is necessary in case _workingBuffer has been resized. (new byte[])
// (The first reference to _workingBuffer goes through the private accessor which
// may initialize it.)

View File

@@ -59,16 +59,16 @@ namespace SharpCompress.Compressors.LZMA
crc32Stream.Dispose();
var compressedCount = _countingWritableSubStream!.Count;
byte[] intBuf = new byte[8];
Span<byte> intBuf = stackalloc byte[8];
BinaryPrimitives.WriteUInt32LittleEndian(intBuf, crc32Stream.Crc);
_countingWritableSubStream.Write(intBuf, 0, 4);
_countingWritableSubStream.Write(intBuf.Slice(0, 4));
BinaryPrimitives.WriteInt64LittleEndian(intBuf, _writeCount);
_countingWritableSubStream.Write(intBuf, 0, 8);
_countingWritableSubStream.Write(intBuf);
//total with headers
BinaryPrimitives.WriteUInt64LittleEndian(intBuf, compressedCount + 6 + 20);
_countingWritableSubStream.Write(intBuf, 0, 8);
_countingWritableSubStream.Write(intBuf);
}
_finished = true;
}
@@ -118,7 +118,7 @@ namespace SharpCompress.Compressors.LZMA
public override void SetLength(long value) => throw new NotImplementedException();
#if !NET461 && !NETSTANDARD2_0
#if !NETFRAMEWORK && !NETSTANDARD2_0
public override int Read(Span<byte> buffer)
{

View File

@@ -31,7 +31,11 @@ namespace SharpCompress.Compressors.PPMd
public PpmdVersion Version { get; } = PpmdVersion.I1;
internal ModelRestorationMethod RestorationMethod { get; }
public PpmdProperties(byte[] properties)
public PpmdProperties(byte[] properties) : this(properties.AsSpan())
{
}
public PpmdProperties(ReadOnlySpan<byte> properties)
{
if (properties.Length == 2)
{
@@ -43,7 +47,7 @@ namespace SharpCompress.Compressors.PPMd
else if (properties.Length == 5)
{
Version = PpmdVersion.H7Z;
AllocatorSize = BinaryPrimitives.ReadInt32LittleEndian(properties.AsSpan(1));
AllocatorSize = BinaryPrimitives.ReadInt32LittleEndian(properties.Slice(1));
ModelOrder = properties[0];
}
}

View File

@@ -1,4 +1,5 @@
using System;
using System.Buffers.Binary;
using System.IO;
namespace SharpCompress.Compressors.Xz
@@ -8,7 +9,7 @@ namespace SharpCompress.Compressors.Xz
public static int ReadLittleEndianInt32(this BinaryReader reader)
{
byte[] bytes = reader.ReadBytes(4);
return (bytes[0] + (bytes[1] << 8) + (bytes[2] << 16) + (bytes[3] << 24));
return BinaryPrimitives.ReadInt32LittleEndian(bytes);
}
internal static uint ReadLittleEndianUInt32(this BinaryReader reader)
@@ -17,13 +18,13 @@ namespace SharpCompress.Compressors.Xz
}
public static int ReadLittleEndianInt32(this Stream stream)
{
byte[] bytes = new byte[4];
Span<byte> bytes = stackalloc byte[4];
var read = stream.ReadFully(bytes);
if (!read)
{
throw new EndOfStreamException();
}
return (bytes[0] + (bytes[1] << 8) + (bytes[2] << 16) + (bytes[3] << 24));
return BinaryPrimitives.ReadInt32LittleEndian(bytes);
}
internal static uint ReadLittleEndianUInt32(this Stream stream)

View File

@@ -1,7 +1,6 @@
#nullable disable
using System;
using System.Collections.Generic;
namespace SharpCompress.Compressors.Xz
{
@@ -24,7 +23,7 @@ namespace SharpCompress.Compressors.Xz
public static UInt32 Compute(UInt32 polynomial, UInt32 seed, byte[] buffer)
{
return ~CalculateHash(InitializeTable(polynomial), seed, buffer, 0, buffer.Length);
return ~CalculateHash(InitializeTable(polynomial), seed, buffer);
}
private static UInt32[] InitializeTable(UInt32 polynomial)
@@ -61,16 +60,16 @@ namespace SharpCompress.Compressors.Xz
return createTable;
}
private static UInt32 CalculateHash(UInt32[] table, UInt32 seed, IList<byte> buffer, int start, int size)
private static UInt32 CalculateHash(UInt32[] table, UInt32 seed, ReadOnlySpan<byte> buffer)
{
var crc = seed;
for (var i = start; i < size - start; i++)
int len = buffer.Length;
for (var i = 0; i < len; i++)
{
crc = (crc >> 8) ^ table[buffer[i] ^ crc & 0xff];
crc = (crc >> 8) ^ table[(buffer[i] ^ crc) & 0xff];
}
return crc;
}
}
}

View File

@@ -22,14 +22,14 @@ namespace SharpCompress.Compressors.Xz
{
Table ??= CreateTable(Iso3309Polynomial);
return CalculateHash(seed, Table, buffer, 0, buffer.Length);
return CalculateHash(seed, Table, buffer);
}
public static UInt64 CalculateHash(UInt64 seed, UInt64[] table, IList<byte> buffer, int start, int size)
public static UInt64 CalculateHash(UInt64 seed, UInt64[] table, ReadOnlySpan<byte> buffer)
{
var crc = seed;
for (var i = start; i < size; i++)
int len = buffer.Length;
for (var i = 0; i < len; i++)
{
unchecked
{

View File

@@ -0,0 +1,201 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public class CompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] outputBuffer;
private Compressor compressor;
private ZSTD_outBuffer_s output;
public CompressionStream(Stream stream, int level = Compressor.DefaultCompressionLevel,
int bufferSize = 0)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanWrite)
throw new ArgumentException("Stream is not writable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
compressor = new Compressor(level);
var outputBufferSize =
bufferSize > 0 ? bufferSize : (int) Methods.ZSTD_CStreamOutSize().EnsureZstdSuccess();
outputBuffer = ArrayPool<byte>.Shared.Rent(outputBufferSize);
output = new ZSTD_outBuffer_s {pos = 0, size = (nuint) outputBufferSize};
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
EnsureNotDisposed();
compressor.SetParameter(parameter, value);
}
public int GetParameter(ZSTD_cParameter parameter)
{
EnsureNotDisposed();
return compressor.GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
compressor.LoadDictionary(dict);
}
~CompressionStream() => Dispose(false);
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask DisposeAsync()
#else
public async ValueTask DisposeAsync()
#endif
{
if (compressor == null)
return;
try
{
await FlushAsync().ConfigureAwait(false);
}
finally
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
}
protected override void Dispose(bool disposing)
{
if (compressor == null)
return;
try
{
if (disposing)
Flush();
}
finally
{
ReleaseUnmanagedResources();
}
}
private void ReleaseUnmanagedResources()
{
compressor.Dispose();
ArrayPool<byte>.Shared.Return(outputBuffer);
}
public override void Flush()
=> WriteInternal(null, true);
public override async Task FlushAsync(CancellationToken cancellationToken)
=> await WriteInternalAsync(null, true, cancellationToken).ConfigureAwait(false);
public override void Write(byte[] buffer, int offset, int count)
=> Write(new ReadOnlySpan<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NET461
public override void Write(ReadOnlySpan<byte> buffer)
=> WriteInternal(buffer, false);
#else
public void Write(ReadOnlySpan<byte> buffer)
=> WriteInternal(buffer, false);
#endif
private void WriteInternal(ReadOnlySpan<byte> buffer, bool lastChunk)
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s {pos = 0, size = buffer != null ? (nuint) buffer.Length : 0};
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(ref input, buffer,
lastChunk ? ZSTD_EndDirective.ZSTD_e_end : ZSTD_EndDirective.ZSTD_e_continue);
var written = (int) output.pos;
if (written > 0)
innerStream.Write(outputBuffer, 0, written);
} while (lastChunk ? remaining > 0 : input.pos < input.size);
}
private async ValueTask WriteInternalAsync(ReadOnlyMemory<byte>? buffer, bool lastChunk,
CancellationToken cancellationToken = default)
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s { pos = 0, size = buffer.HasValue ? (nuint)buffer.Value.Length : 0 };
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(ref input, buffer.HasValue ? buffer.Value.Span : null,
lastChunk ? ZSTD_EndDirective.ZSTD_e_end : ZSTD_EndDirective.ZSTD_e_continue);
var written = (int) output.pos;
if (written > 0)
await innerStream.WriteAsync(outputBuffer, 0, written, cancellationToken).ConfigureAwait(false);
} while (lastChunk ? remaining > 0 : input.pos < input.size);
}
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken).AsTask();
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default)
=> await WriteInternalAsync(buffer, false, cancellationToken).ConfigureAwait(false);
#else
public async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default)
=> await WriteInternalAsync(buffer, false, cancellationToken).ConfigureAwait(false);
#endif
internal unsafe nuint CompressStream(ref ZSTD_inBuffer_s input, ReadOnlySpan<byte> inputBuffer,
ZSTD_EndDirective directive)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return compressor.CompressStream(ref input, ref output, directive).EnsureZstdSuccess();
}
}
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (compressor == null)
throw new ObjectDisposedException(nameof(CompressionStream));
}
}
}

View File

@@ -0,0 +1,164 @@
using System;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public unsafe class Compressor : IDisposable
{
public static int MinCompressionLevel => Methods.ZSTD_minCLevel();
public static int MaxCompressionLevel => Methods.ZSTD_maxCLevel();
public const int DefaultCompressionLevel = 0;
private int level = DefaultCompressionLevel;
private ZSTD_CCtx_s* cctx;
public int Level
{
get => level;
set
{
if (level != value)
{
level = value;
SetParameter(ZSTD_cParameter.ZSTD_c_compressionLevel, value);
}
}
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
EnsureNotDisposed();
Methods.ZSTD_CCtx_setParameter(cctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_cParameter parameter)
{
EnsureNotDisposed();
int value;
Methods.ZSTD_CCtx_getParameter(cctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
if (dict == null)
{
Methods.ZSTD_CCtx_loadDictionary(cctx, null, 0).EnsureZstdSuccess();
}
else
{
fixed (byte* dictPtr = dict)
Methods.ZSTD_CCtx_loadDictionary(cctx, dictPtr, (nuint) dict.Length).EnsureZstdSuccess();
}
}
public Compressor(int level = DefaultCompressionLevel)
{
cctx = Methods.ZSTD_createCCtx();
if (cctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create cctx");
Level = level;
}
~Compressor()
{
ReleaseUnmanagedResources();
}
public static int GetCompressBound(int length)
=> (int) Methods.ZSTD_compressBound((nuint) length);
public static ulong GetCompressBoundLong(ulong length)
=> Methods.ZSTD_compressBound((nuint) length);
public Span<byte> Wrap(ReadOnlySpan<byte> src)
{
var dest = new byte[GetCompressBound(src.Length)];
var length = Wrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Wrap(byte[] src, byte[] dest, int offset)
=> Wrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Wrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
return (int) Methods
.ZSTD_compress2(cctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length)
.EnsureZstdSuccess();
}
public int Wrap(ArraySegment<byte> src, ArraySegment<byte> dest)
=> Wrap((ReadOnlySpan<byte>) src, dest);
public int Wrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength)
=> Wrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength));
public bool TryWrap(byte[] src, byte[] dest, int offset, out int written)
=> TryWrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryWrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
var returnValue =
Methods.ZSTD_compress2(cctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length);
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int) returnValue;
return true;
}
}
public bool TryWrap(ArraySegment<byte> src, ArraySegment<byte> dest, out int written)
=> TryWrap((ReadOnlySpan<byte>)src, dest, out written);
public bool TryWrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength, out int written)
=> TryWrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength), out written);
private void ReleaseUnmanagedResources()
{
if (cctx != null)
{
Methods.ZSTD_freeCCtx(cctx);
cctx = null;
}
}
public void Dispose()
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
private void EnsureNotDisposed()
{
if (cctx == null)
throw new ObjectDisposedException(nameof(Compressor));
}
internal nuint CompressStream(ref ZSTD_inBuffer_s input, ref ZSTD_outBuffer_s output, ZSTD_EndDirective directive)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
return Methods.ZSTD_compressStream2(cctx, outputPtr, inputPtr, directive).EnsureZstdSuccess();
}
}
}
}

View File

@@ -0,0 +1,9 @@
namespace ZstdSharp
{
internal class Constants
{
//NOTE: https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element#remarks
//NOTE: https://github.com/dotnet/runtime/blob/v5.0.0-rtm.20519.4/src/libraries/System.Private.CoreLib/src/System/Array.cs#L27
public const ulong MaxByteArrayLength = 0x7FFFFFC7;
}
}

View File

@@ -0,0 +1,190 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public class DecompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] inputBuffer;
private readonly int inputBufferSize;
private Decompressor decompressor;
private ZSTD_inBuffer_s input;
private nuint lastDecompressResult = 0;
public DecompressionStream(Stream stream, int bufferSize = 0)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanRead)
throw new ArgumentException("Stream is not readable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
decompressor = new Decompressor();
inputBufferSize = bufferSize > 0 ? bufferSize : (int) Methods.ZSTD_CStreamInSize().EnsureZstdSuccess();
inputBuffer = ArrayPool<byte>.Shared.Rent(inputBufferSize);
input = new ZSTD_inBuffer_s {pos = (nuint) inputBufferSize, size = (nuint) inputBufferSize};
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
EnsureNotDisposed();
decompressor.SetParameter(parameter, value);
}
public int GetParameter(ZSTD_dParameter parameter)
{
EnsureNotDisposed();
return decompressor.GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
decompressor.LoadDictionary(dict);
}
~DecompressionStream() => Dispose(false);
protected override void Dispose(bool disposing)
{
if (decompressor == null)
return;
decompressor.Dispose();
ArrayPool<byte>.Shared.Return(inputBuffer);
}
public override int Read(byte[] buffer, int offset, int count)
=> Read(new Span<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override int Read(Span<byte> buffer)
#else
public int Read(Span<byte> buffer)
#endif
{
EnsureNotDisposed();
var output = new ZSTD_outBuffer_s {pos = 0, size = (nuint) buffer.Length};
while (output.pos < output.size)
{
if (input.pos >= input.size)
{
int bytesRead;
if ((bytesRead = innerStream.Read(inputBuffer, 0, inputBufferSize)) == 0)
{
if (lastDecompressResult != 0)
throw new EndOfStreamException("Premature end of stream");
break;
}
input.size = (nuint) bytesRead;
input.pos = 0;
}
lastDecompressResult = DecompressStream(ref output, buffer);
}
return (int) output.pos;
}
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> ReadAsync(new Memory<byte>(buffer, offset, count), cancellationToken).AsTask();
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask<int> ReadAsync(Memory<byte> buffer,
CancellationToken cancellationToken = default)
#else
public async ValueTask<int> ReadAsync(Memory<byte> buffer,
CancellationToken cancellationToken = default)
#endif
{
EnsureNotDisposed();
var output = new ZSTD_outBuffer_s { pos = 0, size = (nuint)buffer.Length};
while (output.pos < output.size)
{
if (input.pos >= input.size)
{
int bytesRead;
if ((bytesRead = await innerStream.ReadAsync(inputBuffer, 0, inputBufferSize, cancellationToken)
.ConfigureAwait(false)) == 0)
{
if (lastDecompressResult != 0)
throw new EndOfStreamException("Premature end of stream");
break;
}
input.size = (nuint) bytesRead;
input.pos = 0;
}
lastDecompressResult = DecompressStream(ref output, buffer.Span);
}
return (int) output.pos;
}
private unsafe nuint DecompressStream(ref ZSTD_outBuffer_s output, Span<byte> outputBuffer)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return decompressor.DecompressStream(ref input, ref output);
}
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Flush() => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (decompressor == null)
throw new ObjectDisposedException(nameof(DecompressionStream));
}
#if NETSTANDARD2_0 || NET461
public virtual ValueTask DisposeAsync()
{
try
{
Dispose();
return default;
}
catch (Exception exc)
{
return new ValueTask(Task.FromException(exc));
}
}
#endif
}
}

View File

@@ -0,0 +1,149 @@
using System;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public unsafe class Decompressor : IDisposable
{
private ZSTD_DCtx_s* dctx;
public Decompressor()
{
dctx = Methods.ZSTD_createDCtx();
if (dctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create dctx");
}
~Decompressor()
{
ReleaseUnmanagedResources();
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
EnsureNotDisposed();
Methods.ZSTD_DCtx_setParameter(dctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_dParameter parameter)
{
EnsureNotDisposed();
int value;
Methods.ZSTD_DCtx_getParameter(dctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
if (dict == null)
{
Methods.ZSTD_DCtx_loadDictionary(dctx, null, 0).EnsureZstdSuccess();
}
else
{
fixed (byte* dictPtr = dict)
Methods.ZSTD_DCtx_loadDictionary(dctx, dictPtr, (nuint) dict.Length).EnsureZstdSuccess();
}
}
public static ulong GetDecompressedSize(ReadOnlySpan<byte> src)
{
fixed (byte* srcPtr = src)
return Methods.ZSTD_decompressBound(srcPtr, (nuint) src.Length).EnsureContentSizeOk();
}
public static ulong GetDecompressedSize(ArraySegment<byte> src)
=> GetDecompressedSize((ReadOnlySpan<byte>) src);
public static ulong GetDecompressedSize(byte[] src, int srcOffset, int srcLength)
=> GetDecompressedSize(new ReadOnlySpan<byte>(src, srcOffset, srcLength));
public Span<byte> Unwrap(ReadOnlySpan<byte> src, int maxDecompressedSize = int.MaxValue)
{
var expectedDstSize = GetDecompressedSize(src);
if (expectedDstSize > (ulong) maxDecompressedSize)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than {nameof(maxDecompressedSize)} {maxDecompressedSize}");
if (expectedDstSize > Constants.MaxByteArrayLength)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than max possible byte array size {Constants.MaxByteArrayLength}");
var dest = new byte[expectedDstSize];
var length = Unwrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Unwrap(byte[] src, byte[] dest, int offset)
=> Unwrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Unwrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
return (int) Methods
.ZSTD_decompressDCtx(dctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length)
.EnsureZstdSuccess();
}
public int Unwrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength)
=> Unwrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength));
public bool TryUnwrap(byte[] src, byte[] dest, int offset, out int written)
=> TryUnwrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryUnwrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
var returnValue =
Methods.ZSTD_decompressDCtx(dctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length);
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int) returnValue;
return true;
}
}
public bool TryUnwrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength, out int written)
=> TryUnwrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength), out written);
private void ReleaseUnmanagedResources()
{
if (dctx != null)
{
Methods.ZSTD_freeDCtx(dctx);
dctx = null;
}
}
public void Dispose()
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
private void EnsureNotDisposed()
{
if (dctx == null)
throw new ObjectDisposedException(nameof(Decompressor));
}
internal nuint DecompressStream(ref ZSTD_inBuffer_s input, ref ZSTD_outBuffer_s output)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
return Methods.ZSTD_decompressStream(dctx, outputPtr, inputPtr).EnsureZstdSuccess();
}
}
}
}

View File

@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public static unsafe class DictBuilder
{
public static byte[] TrainFromBuffer(IEnumerable<byte[]> samples, int dictCapacity = DefaultDictCapacity)
{
var ms = new MemoryStream();
var samplesSizes = samples.Select(sample =>
{
ms.Write(sample, 0, sample.Length);
return (nuint) sample.Length;
}).ToArray();
var dictBuffer = new byte[dictCapacity];
fixed (byte* dictBufferPtr = dictBuffer)
fixed (byte* samplesBufferPtr = ms.GetBuffer())
fixed (nuint* samplesSizesPtr = samplesSizes)
{
var dictSize = (int) Methods
.ZDICT_trainFromBuffer(dictBufferPtr, (nuint) dictCapacity, samplesBufferPtr, samplesSizesPtr,
(uint) samplesSizes.Length)
.EnsureZdictSuccess();
if (dictCapacity != dictSize)
Array.Resize(ref dictBuffer, dictSize);
return dictBuffer;
}
}
public const int DefaultDictCapacity = 112640; // Used by zstd utility by default
}
}

View File

@@ -0,0 +1,201 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
// Some routines inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson:
// http://graphics.stanford.edu/~seander/bithacks.html
namespace System.Numerics
{
/// <summary>
/// Utility methods for intrinsic bit-twiddling operations.
/// The methods use hardware intrinsics when available on the underlying platform,
/// otherwise they use optimized software fallbacks.
/// </summary>
public static unsafe class BitOperations
{
// hack: should be public because of inline
public static readonly byte* TrailingZeroCountDeBruijn = GetArrayPointer(new byte[]
{
00, 01, 28, 02, 29, 14, 24, 03,
30, 22, 20, 15, 25, 17, 04, 08,
31, 27, 13, 23, 21, 19, 16, 07,
26, 12, 18, 06, 11, 05, 10, 09
});
// hack: should be public because of inline
public static readonly byte* Log2DeBruijn = GetArrayPointer(new byte[]
{
00, 09, 01, 10, 13, 21, 02, 29,
11, 14, 16, 18, 22, 25, 03, 30,
08, 12, 20, 28, 15, 17, 24, 07,
19, 27, 23, 06, 26, 05, 04, 31
});
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(uint value)
{
// The 0->0 contract is fulfilled by setting the LSB to 1.
// Log(1) is 0, and setting the LSB for values > 1 does not change the log2 result.
value |= 1;
// value lzcnt actual expected
// ..0001 31 31-31 0
// ..0010 30 31-30 1
// 0010.. 2 31-2 29
// 0100.. 1 31-1 30
// 1000.. 0 31-0 31
// Fallback contract is 0->0
// No AggressiveInlining due to large method size
// Has conventional contract 0->0 (Log(0) is undefined)
// Fill trailing zeros with ones, eg 00010010 becomes 00011111
value |= value >> 01;
value |= value >> 02;
value |= value >> 04;
value |= value >> 08;
value |= value >> 16;
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return Log2DeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_1100_0100_1010_1100_1101_1101u
(int)((value * 0x07C4ACDDu) >> 27)];
}
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(ulong value)
{
value |= 1;
uint hi = (uint)(value >> 32);
if (hi == 0)
{
return Log2((uint)value);
}
return 32 + Log2(hi);
}
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(int value)
=> TrailingZeroCount((uint)value);
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(uint value)
{
// Unguarded fallback contract is 0->0, BSF contract is 0->undefined
if (value == 0)
{
return 32;
}
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return TrailingZeroCountDeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_0111_1100_1011_0101_0011_0001u
(int)(((value & (uint)-(int)value) * 0x077CB531u) >> 27)]; // Multi-cast mitigates redundant conv.u8
}
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(long value)
=> TrailingZeroCount((ulong)value);
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(ulong value)
{
uint lo = (uint)value;
if (lo == 0)
{
return 32 + TrailingZeroCount((uint)(value >> 32));
}
return TrailingZeroCount(lo);
}
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateLeft(uint value, int offset)
=> (value << offset) | (value >> (32 - offset));
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateLeft(ulong value, int offset)
=> (value << offset) | (value >> (64 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateRight(uint value, int offset)
=> (value >> offset) | (value << (32 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateRight(ulong value, int offset)
=> (value >> offset) | (value << (64 - offset));
}
}
#endif

View File

@@ -0,0 +1,25 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NET5_0_OR_GREATER
namespace System.Runtime.CompilerServices
{
/// <summary>
/// Used to indicate to the compiler that the <c>.locals init</c> flag should not be set in method headers.
/// </summary>
/// <remarks>Internal copy of the .NET 5 attribute.</remarks>
[AttributeUsage(
AttributeTargets.Module |
AttributeTargets.Class |
AttributeTargets.Struct |
AttributeTargets.Interface |
AttributeTargets.Constructor |
AttributeTargets.Method |
AttributeTargets.Property |
AttributeTargets.Event,
Inherited = false)]
internal sealed class SkipLocalsInitAttribute : Attribute
{
}
}
#endif

View File

@@ -0,0 +1,54 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
namespace System.Runtime.Intrinsics
{
public static class Vector128
{
internal const int Size = 16;
public static unsafe Vector128<byte> Create(byte value)
{
byte* pResult = stackalloc byte[16]
{
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
};
return Unsafe.AsRef<Vector128<byte>>(pResult);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<U> As<T, U>(this Vector128<T> vector)
where T : struct
where U : struct =>
Unsafe.As<Vector128<T>, Vector128<U>>(ref vector);
public static T GetElement<T>(this Vector128<T> vector, int index)
where T : struct
{
ref T e0 = ref Unsafe.As<Vector128<T>, T>(ref vector);
return Unsafe.Add(ref e0, index);
}
}
}
#endif

View File

@@ -0,0 +1,36 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace System.Runtime.Intrinsics
{
[StructLayout(LayoutKind.Sequential, Size = Vector128.Size)]
public readonly struct Vector128<T> : IEquatable<Vector128<T>>
where T : struct
{
private readonly ulong _00;
private readonly ulong _01;
public static int Count => Vector128.Size / Unsafe.SizeOf<T>();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool Equals(Vector128<T> other)
{
for (int i = 0; i < Count; i++)
{
if (!((IEquatable<T>)(this.GetElement(i))).Equals(other.GetElement(i)))
{
return false;
}
}
return true;
}
}
}
#endif

View File

@@ -0,0 +1,43 @@
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public static unsafe class ThrowHelper
{
private const ulong ZSTD_CONTENTSIZE_UNKNOWN = unchecked(0UL - 1);
private const ulong ZSTD_CONTENTSIZE_ERROR = unchecked(0UL - 2);
public static nuint EnsureZstdSuccess(this nuint returnValue)
{
if (Methods.ZSTD_isError(returnValue) != 0)
ThrowException(returnValue, Methods.ZSTD_getErrorName(returnValue));
return returnValue;
}
public static nuint EnsureZdictSuccess(this nuint returnValue)
{
if (Methods.ZDICT_isError(returnValue) != 0)
ThrowException(returnValue, Methods.ZDICT_getErrorName(returnValue));
return returnValue;
}
public static ulong EnsureContentSizeOk(this ulong returnValue)
{
if (returnValue == ZSTD_CONTENTSIZE_UNKNOWN)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Decompressed content size is not specified");
if (returnValue == ZSTD_CONTENTSIZE_ERROR)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Decompressed content size cannot be determined (e.g. invalid magic number, srcSize too small)");
return returnValue;
}
private static void ThrowException(nuint returnValue, string message)
{
var code = 0 - returnValue;
throw new ZstdException((ZSTD_ErrorCode) code, message);
}
}
}

View File

@@ -0,0 +1,368 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
static uint* rtbTable = GetArrayPointer(new uint[8] {
0,
473195,
504333,
520860,
550000,
700000,
750000,
830000,
});
static byte* LL_Code = GetArrayPointer(new byte[64]
{
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
16,
17,
17,
18,
18,
19,
19,
20,
20,
20,
20,
21,
21,
21,
21,
22,
22,
22,
22,
22,
22,
22,
22,
23,
23,
23,
23,
23,
23,
23,
23,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
});
static byte* ML_Code = GetArrayPointer(new byte[128]
{
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
32,
33,
33,
34,
34,
35,
35,
36,
36,
36,
36,
37,
37,
37,
37,
38,
38,
38,
38,
38,
38,
38,
38,
39,
39,
39,
39,
39,
39,
39,
39,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
});
static ulong* srcSizeTiers = GetArrayPointer(new ulong[4]
{
16 * (1 << 10),
128 * (1 << 10),
256 * (1 << 10),
(unchecked(0UL - 1)),
});
static ZSTD_blockCompressor?[][] blockCompressor = new ZSTD_blockCompressor?[4][]
{
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast,
ZSTD_compressBlock_fast,
ZSTD_compressBlock_doubleFast,
ZSTD_compressBlock_greedy,
ZSTD_compressBlock_lazy,
ZSTD_compressBlock_lazy2,
ZSTD_compressBlock_btlazy2,
ZSTD_compressBlock_btopt,
ZSTD_compressBlock_btultra,
ZSTD_compressBlock_btultra2,
},
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_doubleFast_extDict,
ZSTD_compressBlock_greedy_extDict,
ZSTD_compressBlock_lazy_extDict,
ZSTD_compressBlock_lazy2_extDict,
ZSTD_compressBlock_btlazy2_extDict,
ZSTD_compressBlock_btopt_extDict,
ZSTD_compressBlock_btultra_extDict,
ZSTD_compressBlock_btultra_extDict,
},
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast_dictMatchState,
ZSTD_compressBlock_fast_dictMatchState,
ZSTD_compressBlock_doubleFast_dictMatchState,
ZSTD_compressBlock_greedy_dictMatchState,
ZSTD_compressBlock_lazy_dictMatchState,
ZSTD_compressBlock_lazy2_dictMatchState,
ZSTD_compressBlock_btlazy2_dictMatchState,
ZSTD_compressBlock_btopt_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
},
new ZSTD_blockCompressor?[10]
{
null,
null,
null,
ZSTD_compressBlock_greedy_dedicatedDictSearch,
ZSTD_compressBlock_lazy_dedicatedDictSearch,
ZSTD_compressBlock_lazy2_dedicatedDictSearch,
null,
null,
null,
null,
},
};
static ZSTD_blockCompressor[][] rowBasedBlockCompressors = new ZSTD_blockCompressor[4][]
{
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_row,
ZSTD_compressBlock_lazy_row,
ZSTD_compressBlock_lazy2_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_extDict_row,
ZSTD_compressBlock_lazy_extDict_row,
ZSTD_compressBlock_lazy2_extDict_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_dictMatchState_row,
ZSTD_compressBlock_lazy_dictMatchState_row,
ZSTD_compressBlock_lazy2_dictMatchState_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy2_dedicatedDictSearch_row,
},
};
static searchMax_f?[][] searchFuncs = new searchMax_f?[4][]
{
new searchMax_f[3]
{
ZSTD_HcFindBestMatch_selectMLS,
ZSTD_BtFindBestMatch_selectMLS,
ZSTD_RowFindBestMatch_selectRowLog,
},
new searchMax_f?[3]
{
null,
null,
null,
},
new searchMax_f[3]
{
ZSTD_HcFindBestMatch_dictMatchState_selectMLS,
ZSTD_BtFindBestMatch_dictMatchState_selectMLS,
ZSTD_RowFindBestMatch_dictMatchState_selectRowLog,
},
new searchMax_f?[3]
{
ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS,
null,
ZSTD_RowFindBestMatch_dedicatedDictSearch_selectRowLog,
},
};
static searchMax_f[] searchFuncsExtGeneric = new searchMax_f[3]
{
ZSTD_HcFindBestMatch_extDict_selectMLS,
ZSTD_BtFindBestMatch_extDict_selectMLS,
ZSTD_RowFindBestMatch_extDict_selectRowLog,
};
static decompressionAlgo[] decompress = new decompressionAlgo[2]
{
HUF_decompress4X1,
HUF_decompress4X2,
};
static uint* dec32table = GetArrayPointer(new uint[8]
{
0,
1,
2,
1,
4,
4,
4,
4,
});
static int* dec64table = GetArrayPointer(new int[8]
{
8,
8,
8,
7,
8,
9,
10,
11,
});
private static byte* emptyString = GetArrayPointer(new byte[] {0});
}
}

View File

@@ -0,0 +1,24 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
public unsafe partial struct BIT_CStream_t
{
public nuint bitContainer;
public uint bitPos;
public sbyte* startPtr;
public sbyte* ptr;
public sbyte* endPtr;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum BIT_DStream_status
{
BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2,
BIT_DStream_overflow = 3,
}
}

View File

@@ -0,0 +1,20 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
public unsafe partial struct BIT_DStream_t
{
public nuint bitContainer;
public uint bitsConsumed;
public sbyte* ptr;
public sbyte* start;
public sbyte* limitPtr;
}
}

View File

@@ -0,0 +1,420 @@
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*-**************************************************************
* Internal functions
****************************************************************/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static uint BIT_highbit32(uint val)
{
assert(val != 0);
{
return (uint) BitOperations.Log2(val);
}
}
public static uint* BIT_mask = GetArrayPointer(new uint[32]
{
0,
1,
3,
7,
0xF,
0x1F,
0x3F,
0x7F,
0xFF,
0x1FF,
0x3FF,
0x7FF,
0xFFF,
0x1FFF,
0x3FFF,
0x7FFF,
0xFFFF,
0x1FFFF,
0x3FFFF,
0x7FFFF,
0xFFFFF,
0x1FFFFF,
0x3FFFFF,
0x7FFFFF,
0xFFFFFF,
0x1FFFFFF,
0x3FFFFFF,
0x7FFFFFF,
0xFFFFFFF,
0x1FFFFFFF,
0x3FFFFFFF,
0x7FFFFFFF,
});
/*-**************************************************************
* bitStream encoding
****************************************************************/
/*! BIT_initCStream() :
* `dstCapacity` must be > sizeof(size_t)
* @return : 0 if success,
* otherwise an error code (can be tested using ERR_isError()) */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, nuint dstCapacity)
{
bitC->bitContainer = 0;
bitC->bitPos = 0;
bitC->startPtr = (sbyte*)(startPtr);
bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - (nuint)(sizeof(nuint));
if (dstCapacity <= (nuint)(sizeof(nuint)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
return 0;
}
/*! BIT_addBits() :
* can add up to 31 bits into `bitC`.
* Note : does not check for register overflow ! */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBits(BIT_CStream_t* bitC, nuint value, uint nbBits)
{
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
assert(nbBits + bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
bitC->bitContainer |= (value & BIT_mask[nbBits]) << (int)bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_addBitsFast() :
* works only if `value` is _clean_,
* meaning all high bits above nbBits are 0 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBitsFast(BIT_CStream_t* bitC, nuint value, uint nbBits)
{
assert((value >> (int)nbBits) == 0);
assert(nbBits + bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
bitC->bitContainer |= value << (int)bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_flushBitsFast() :
* assumption : bitContainer has not overflowed
* unsafe version; does not check buffer overflow */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBitsFast(BIT_CStream_t* bitC)
{
nuint nbBytes = bitC->bitPos >> 3;
assert(bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
assert(bitC->ptr <= bitC->endPtr);
MEM_writeLEST((void*)bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
bitC->bitPos &= 7;
bitC->bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_flushBits() :
* assumption : bitContainer has not overflowed
* safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow.
* overflow will be revealed later on using BIT_closeCStream() */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBits(BIT_CStream_t* bitC)
{
nuint nbBytes = bitC->bitPos >> 3;
assert(bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
assert(bitC->ptr <= bitC->endPtr);
MEM_writeLEST((void*)bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr)
{
bitC->ptr = bitC->endPtr;
}
bitC->bitPos &= 7;
bitC->bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_closeCStream() :
* @return : size of CStream, in bytes,
* or 0 if it could not fit into dstBuffer */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_closeCStream(BIT_CStream_t* bitC)
{
BIT_addBitsFast(bitC, 1, 1);
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr)
{
return 0;
}
return (nuint)((bitC->ptr - bitC->startPtr) + (((bitC->bitPos > 0)) ? 1 : 0));
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize), or an errorCode if a problem is detected
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initDStream(BIT_DStream_t* bitD, void* srcBuffer, nuint srcSize)
{
if (srcSize < 1)
{
memset((void*)(bitD), (0), ((nuint)(sizeof(BIT_DStream_t))));
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
bitD->start = (sbyte*)(srcBuffer);
bitD->limitPtr = bitD->start + (nuint)(sizeof(nuint));
if (srcSize >= (nuint)(sizeof(nuint)))
{
bitD->ptr = (sbyte*)(srcBuffer) + srcSize - (nuint)(sizeof(nuint));
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
{
byte lastByte = ((byte*)(srcBuffer))[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
}
}
else
{
bitD->ptr = bitD->start;
bitD->bitContainer = *(byte*)(bitD->start);
switch (srcSize)
{
case 7:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[6]) << (int)((nuint)(sizeof(nuint)) * 8 - 16);
}
goto case 6;
case 6:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[5]) << (int)((nuint)(sizeof(nuint)) * 8 - 24);
}
goto case 5;
case 5:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[4]) << (int)((nuint)(sizeof(nuint)) * 8 - 32);
}
goto case 4;
case 4:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[3]) << 24;
}
goto case 3;
case 3:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[2]) << 16;
}
goto case 2;
case 2:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[1]) << 8;
}
goto default;
default:
{
break;
}
}
{
byte lastByte = ((byte*)(srcBuffer))[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
}
bitD->bitsConsumed += (uint)((nuint)(sizeof(nuint)) - srcSize) * 8;
}
return srcSize;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getUpperBits(nuint bitContainer, uint start)
{
return bitContainer >> (int)start;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getMiddleBits(nuint bitContainer, uint start, uint nbBits)
{
uint regMask = (uint)((nuint)(sizeof(nuint)) * 8 - 1);
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
return (bitContainer >> (int)(start & regMask)) & BIT_mask[nbBits];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getLowerBits(nuint bitContainer, uint nbBits)
{
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBits(BIT_DStream_t* bitD, uint nbBits)
{
return BIT_getMiddleBits(bitD->bitContainer, (uint)(((nuint)(sizeof(nuint)) * 8) - bitD->bitsConsumed - nbBits), nbBits);
}
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
uint regMask = (uint)((nuint)(sizeof(nuint)) * 8 - 1);
assert(nbBits >= 1);
return (bitD->bitContainer << (int)(bitD->bitsConsumed & regMask)) >> (int)(((regMask + 1) - nbBits) & regMask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_skipBits(BIT_DStream_t* bitD, uint nbBits)
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBits(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_reloadDStreamFast() :
* Similar to BIT_reloadDStream(), but with two differences:
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
* point you must use BIT_reloadDStream() to reload.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
{
if ((bitD->ptr < bitD->limitPtr))
{
return BIT_DStream_status.BIT_DStream_overflow;
}
assert(bitD->bitsConsumed <= (nuint)(sizeof(nuint)) * 8);
bitD->ptr -= bitD->bitsConsumed >> 3;
bitD->bitsConsumed &= 7;
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
return BIT_DStream_status.BIT_DStream_unfinished;
}
/*! BIT_reloadDStream() :
* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > ((nuint)(sizeof(nuint)) * 8))
{
return BIT_DStream_status.BIT_DStream_overflow;
}
if (bitD->ptr >= bitD->limitPtr)
{
return BIT_reloadDStreamFast(bitD);
}
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < (nuint)(sizeof(nuint)) * 8)
{
return BIT_DStream_status.BIT_DStream_endOfBuffer;
}
return BIT_DStream_status.BIT_DStream_completed;
}
{
uint nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_status.BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (uint)(bitD->ptr - bitD->start);
result = BIT_DStream_status.BIT_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes * 8;
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
return result;
}
}
/*! BIT_endOfDStream() :
* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint BIT_endOfDStream(BIT_DStream_t* DStream)
{
return ((((DStream->ptr == DStream->start) && (DStream->bitsConsumed == (nuint)(sizeof(nuint)) * 8))) ? 1U : 0U);
}
}
}

View File

@@ -0,0 +1,29 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* COVER_best_t is used for two purposes:
* 1. Synchronizing threads.
* 2. Saving the best parameters and dictionary.
*
* All of the methods except COVER_best_init() are thread safe if zstd is
* compiled with multithreaded support.
*/
public unsafe partial struct COVER_best_s
{
public int mutex;
public int cond;
public nuint liveJobs;
public void* dict;
public nuint dictSize;
public ZDICT_cover_params_t parameters;
public nuint compressedSize;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* Struct used for the dictionary selection function.
*/
public unsafe partial struct COVER_dictSelection
{
public byte* dictContent;
public nuint dictSize;
public nuint totalCompressedSize;
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
*Number of epochs and size of each epoch.
*/
public partial struct COVER_epoch_info_t
{
public uint num;
public uint size;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* A segment is a range in the source as well as the score of the segment.
*/
public partial struct COVER_segment_t
{
public uint begin;
public uint end;
public uint score;
}
}

View File

@@ -0,0 +1,411 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
public static int g_displayLevel = 2;
/**
* Returns the sum of the sample sizes.
*/
public static nuint COVER_sum(nuint* samplesSizes, uint nbSamples)
{
nuint sum = 0;
uint i;
for (i = 0; i < nbSamples; ++i)
{
sum += samplesSizes[i];
}
return sum;
}
/**
* Warns the user when their corpus is too small.
*/
public static void COVER_warnOnSmallCorpus(nuint maxDictSize, nuint nbDmers, int displayLevel)
{
double ratio = (double)(nbDmers) / maxDictSize;
if (ratio >= 10)
{
return;
}
if (displayLevel >= 1)
{
;
}
}
/**
* Computes the number of epochs and the size of each epoch.
* We will make sure that each epoch gets at least 10 * k bytes.
*
* The COVER algorithms divide the data up into epochs of equal size and
* select one segment from each epoch.
*
* @param maxDictSize The maximum allowed dictionary size.
* @param nbDmers The number of dmers we are training on.
* @param k The parameter k (segment size).
* @param passes The target number of passes over the dmer corpus.
* More passes means a better dictionary.
*/
public static COVER_epoch_info_t COVER_computeEpochs(uint maxDictSize, uint nbDmers, uint k, uint passes)
{
uint minEpochSize = k * 10;
COVER_epoch_info_t epochs;
epochs.num = (uint)((1) > (maxDictSize / k / passes) ? (1) : (maxDictSize / k / passes));
epochs.size = nbDmers / epochs.num;
if (epochs.size >= minEpochSize)
{
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
epochs.size = ((minEpochSize) < (nbDmers) ? (minEpochSize) : (nbDmers));
epochs.num = nbDmers / epochs.size;
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
/**
* Checks total compressed size of a dictionary
*/
public static nuint COVER_checkTotalCompressedSize(ZDICT_cover_params_t parameters, nuint* samplesSizes, byte* samples, nuint* offsets, nuint nbTrainSamples, nuint nbSamples, byte* dict, nuint dictBufferCapacity)
{
nuint totalCompressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
ZSTD_CCtx_s* cctx;
ZSTD_CDict_s* cdict;
void* dst;
nuint dstCapacity;
nuint i;
{
nuint maxSampleSize = 0;
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
maxSampleSize = ((samplesSizes[i]) > (maxSampleSize) ? (samplesSizes[i]) : (maxSampleSize));
}
dstCapacity = ZSTD_compressBound(maxSampleSize);
dst = malloc(dstCapacity);
}
cctx = ZSTD_createCCtx();
cdict = ZSTD_createCDict((void*)dict, dictBufferCapacity, parameters.zParams.compressionLevel);
if (dst == null || cctx == null || cdict == null)
{
goto _compressCleanup;
}
totalCompressedSize = dictBufferCapacity;
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
nuint size = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, (void*)(samples + offsets[i]), samplesSizes[i], cdict);
if ((ERR_isError(size)) != 0)
{
totalCompressedSize = size;
goto _compressCleanup;
}
totalCompressedSize += size;
}
_compressCleanup:
ZSTD_freeCCtx(cctx);
ZSTD_freeCDict(cdict);
if (dst != null)
{
free(dst);
}
return totalCompressedSize;
}
/**
* Initialize the `COVER_best_t`.
*/
public static void COVER_best_init(COVER_best_s* best)
{
if (best == null)
{
return;
}
best->liveJobs = 0;
best->dict = null;
best->dictSize = 0;
best->compressedSize = unchecked((nuint)(-1));
memset((void*)&best->parameters, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
}
/**
* Wait until liveJobs == 0.
*/
public static void COVER_best_wait(COVER_best_s* best)
{
if (best == null)
{
return;
}
while (best->liveJobs != 0)
{
}
}
/**
* Call COVER_best_wait() and then destroy the COVER_best_t.
*/
public static void COVER_best_destroy(COVER_best_s* best)
{
if (best == null)
{
return;
}
COVER_best_wait(best);
if (best->dict != null)
{
free(best->dict);
}
}
/**
* Called when a thread is about to be launched.
* Increments liveJobs.
*/
public static void COVER_best_start(COVER_best_s* best)
{
if (best == null)
{
return;
}
++best->liveJobs;
}
/**
* Called when a thread finishes executing, both on error or success.
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
public static void COVER_best_finish(COVER_best_s* best, ZDICT_cover_params_t parameters, COVER_dictSelection selection)
{
void* dict = (void*)selection.dictContent;
nuint compressedSize = selection.totalCompressedSize;
nuint dictSize = selection.dictSize;
if (best == null)
{
return;
}
{
nuint liveJobs;
--best->liveJobs;
liveJobs = best->liveJobs;
if (compressedSize < best->compressedSize)
{
if (best->dict == null || best->dictSize < dictSize)
{
if (best->dict != null)
{
free(best->dict);
}
best->dict = malloc(dictSize);
if (best->dict == null)
{
best->compressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
best->dictSize = 0;
return;
}
}
if (dict != null)
{
memcpy(best->dict, dict, dictSize);
best->dictSize = dictSize;
best->parameters = parameters;
best->compressedSize = compressedSize;
}
}
if (liveJobs == 0)
{
}
}
}
/**
* Error function for COVER_selectDict function. Returns a struct where
* return.totalCompressedSize is a ZSTD error.
*/
public static COVER_dictSelection COVER_dictSelectionError(nuint error)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = null,
dictSize = 0,
totalCompressedSize = error,
};
return selection;
}
/**
* Error function for COVER_selectDict function. Checks if the return
* value is an error.
*/
public static uint COVER_dictSelectionIsError(COVER_dictSelection selection)
{
return ((((ERR_isError(selection.totalCompressedSize)) != 0 || selection.dictContent == null)) ? 1U : 0U);
}
/**
* Always call after selectDict is called to free up used memory from
* newly created dictionary.
*/
public static void COVER_dictSelectionFree(COVER_dictSelection selection)
{
free((void*)selection.dictContent);
}
/**
* Called to finalize the dictionary and select one based on whether or not
* the shrink-dict flag was enabled. If enabled the dictionary used is the
* smallest dictionary within a specified regression of the compressed size
* from the largest dictionary.
*/
public static COVER_dictSelection COVER_selectDict(byte* customDictContent, nuint dictBufferCapacity, nuint dictContentSize, byte* samplesBuffer, nuint* samplesSizes, uint nbFinalizeSamples, nuint nbCheckSamples, nuint nbSamples, ZDICT_cover_params_t @params, nuint* offsets, nuint totalCompressedSize)
{
nuint largestDict = 0;
nuint largestCompressed = 0;
byte* customDictContentEnd = customDictContent + dictContentSize;
byte* largestDictbuffer = (byte*)(malloc(dictBufferCapacity));
byte* candidateDictBuffer = (byte*)(malloc(dictBufferCapacity));
double regressionTolerance = ((double)(@params.shrinkDictMaxRegression) / 100.0) + 1.00;
if (largestDictbuffer == null || candidateDictBuffer == null)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
memcpy((void*)largestDictbuffer, (void*)customDictContent, dictContentSize);
dictContentSize = ZDICT_finalizeDictionary((void*)largestDictbuffer, dictBufferCapacity, (void*)customDictContent, dictContentSize, (void*)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, largestDictbuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (@params.shrinkDict == 0)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)candidateDictBuffer);
return selection;
}
largestDict = dictContentSize;
largestCompressed = totalCompressedSize;
dictContentSize = 256;
while (dictContentSize < largestDict)
{
memcpy((void*)candidateDictBuffer, (void*)largestDictbuffer, largestDict);
dictContentSize = ZDICT_finalizeDictionary((void*)candidateDictBuffer, dictBufferCapacity, (void*)(customDictContentEnd - dictContentSize), dictContentSize, (void*)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, candidateDictBuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (totalCompressedSize <= largestCompressed * regressionTolerance)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = candidateDictBuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)largestDictbuffer);
return selection;
}
dictContentSize *= 2;
}
dictContentSize = largestDict;
totalCompressedSize = largestCompressed;
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)candidateDictBuffer);
return selection;
}
}
}
}

View File

@@ -0,0 +1,18 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-***************************/
/* generic DTableDesc */
/*-***************************/
public partial struct DTableDesc
{
public byte maxTableLog;
public byte tableType;
public byte tableLog;
public byte reserved;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct EStats_ress_t
{
/* dictionary */
public ZSTD_CDict_s* dict;
/* working context */
public ZSTD_CCtx_s* zc;
/* must be ZSTD_BLOCKSIZE_MAX allocated */
public void* workPlace;
}
}

View File

@@ -0,0 +1,425 @@
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*=== Version ===*/
public static uint FSE_versionNumber()
{
return (uint)((0 * 100 * 100 + 9 * 100 + 0));
}
/*=== Error Management ===*/
public static uint FSE_isError(nuint code)
{
return ERR_isError(code);
}
public static string FSE_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/* Error Management */
public static uint HUF_isError(nuint code)
{
return ERR_isError(code);
}
public static string HUF_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
[InlineMethod.Inline]
private static uint FSE_ctz(uint val)
{
assert(val != 0);
{
return (uint) BitOperations.TrailingZeroCount(val);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_readNCount_body(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
byte* istart = (byte*)(headerBuffer);
byte* iend = istart + hbSize;
byte* ip = istart;
int nbBits;
int remaining;
int threshold;
uint bitStream;
int bitCount;
uint charnum = 0;
uint maxSV1 = *maxSVPtr + 1;
int previous0 = 0;
if (hbSize < 8)
{
sbyte* buffer = stackalloc sbyte[8];
memset(buffer, 0, sizeof(sbyte) * 8);
memcpy((void*)(buffer), (headerBuffer), (hbSize));
{
nuint countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, (void*)buffer, (nuint)(8));
if ((FSE_isError(countSize)) != 0)
{
return countSize;
}
if (countSize > hbSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
return countSize;
}
}
assert(hbSize >= 8);
memset((void*)(normalizedCounter), (0), ((*maxSVPtr + 1) * (nuint)(sizeof(short))));
bitStream = MEM_readLE32((void*)ip);
nbBits = (int)((bitStream & 0xF) + 5);
if (nbBits > 15)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
bitStream >>= 4;
bitCount = 4;
*tableLogPtr = (uint)nbBits;
remaining = (1 << nbBits) + 1;
threshold = 1 << nbBits;
nbBits++;
for (;;)
{
if (previous0 != 0)
{
int repeats = (int)(FSE_ctz(~bitStream | 0x80000000) >> 1);
while (repeats >= 12)
{
charnum += (uint)(3 * 12);
if ((ip <= iend - 7))
{
ip += 3;
}
else
{
bitCount -= (int)(8 * (iend - 7 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
repeats = (int)(FSE_ctz(~bitStream | 0x80000000) >> 1);
}
charnum += (uint)(3 * repeats);
bitStream >>= 2 * repeats;
bitCount += 2 * repeats;
assert((bitStream & 3) < 3);
charnum += bitStream & 3;
bitCount += 2;
if (charnum >= maxSV1)
{
break;
}
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4))
{
assert((bitCount >> 3) <= 3);
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
}
{
int max = (2 * threshold - 1) - remaining;
int count;
if ((bitStream & (uint)((threshold - 1))) < (uint)(max))
{
count = (int)(bitStream & (uint)((threshold - 1)));
bitCount += nbBits - 1;
}
else
{
count = (int)(bitStream & (uint)((2 * threshold - 1)));
if (count >= threshold)
{
count -= max;
}
bitCount += nbBits;
}
count--;
if (count >= 0)
{
remaining -= count;
}
else
{
assert(count == -1);
remaining += count;
}
normalizedCounter[charnum++] = (short)(count);
previous0 = (count == 0 ? 1 : 0);
assert(threshold > 1);
if (remaining < threshold)
{
if (remaining <= 1)
{
break;
}
nbBits = (int)(BIT_highbit32((uint)remaining) + 1);
threshold = 1 << (nbBits - 1);
}
if (charnum >= maxSV1)
{
break;
}
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4))
{
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
}
}
if (remaining != 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
if (charnum > maxSV1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall)));
}
if (bitCount > 32)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*maxSVPtr = charnum - 1;
ip += (bitCount + 7) >> 3;
return (nuint)(ip - istart);
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint FSE_readNCount_body_default(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
private static nuint FSE_readNCount_body_bmi2(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
/*! FSE_readNCount_bmi2():
* Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
*/
public static nuint FSE_readNCount_bmi2(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize, int bmi2)
{
if (bmi2 != 0)
{
return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
/*! FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
public static nuint FSE_readNCount(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, 0);
}
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
public static nuint HUF_readStats(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize)
{
uint* wksp = stackalloc uint[218];
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, (void*)wksp, (nuint)(sizeof(uint) * 218), 0);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint HUF_readStats_body(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize, int bmi2)
{
uint weightTotal;
byte* ip = (byte*)(src);
nuint iSize;
nuint oSize;
if (srcSize == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
iSize = ip[0];
if (iSize >= 128)
{
oSize = iSize - 127;
iSize = ((oSize + 1) / 2);
if (iSize + 1 > srcSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (oSize >= hwSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
ip += 1;
{
uint n;
for (n = 0; n < oSize; n += 2)
{
huffWeight[n] = (byte)(ip[n / 2] >> 4);
huffWeight[n + 1] = (byte)(ip[n / 2] & 15);
}
}
}
else
{
if (iSize + 1 > srcSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
oSize = FSE_decompress_wksp_bmi2((void*)huffWeight, hwSize - 1, (void*)(ip + 1), iSize, 6, workSpace, wkspSize, bmi2);
if ((FSE_isError(oSize)) != 0)
{
return oSize;
}
}
memset((void*)(rankStats), (0), ((uint)((12 + 1)) * (nuint)(sizeof(uint))));
weightTotal = 0;
{
uint n;
for (n = 0; n < oSize; n++)
{
if (huffWeight[n] >= 12)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
rankStats[huffWeight[n]]++;
weightTotal += (uint)((1 << (int)(huffWeight[n])) >> 1);
}
}
if (weightTotal == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
{
uint tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > 12)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*tableLogPtr = tableLog;
{
uint total = (uint)(1 << (int)tableLog);
uint rest = total - weightTotal;
uint verif = (uint)(1 << (int)(BIT_highbit32(rest)));
uint lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
huffWeight[oSize] = (byte)(lastWeight);
rankStats[lastWeight]++;
}
}
if ((rankStats[1] < 2) || (rankStats[1] & 1) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*nbSymbolsPtr = (uint)(oSize + 1);
return iSize + 1;
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint HUF_readStats_body_default(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize)
{
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
}
private static nuint HUF_readStats_body_bmi2(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize)
{
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
}
public static nuint HUF_readStats_wksp(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize, int bmi2)
{
if (bmi2 != 0)
{
return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
}
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
}
}
}

View File

@@ -0,0 +1,184 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ERR_isError(nuint code)
{
return (((code > (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxCode))))) ? 1U : 0U);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ZSTD_ErrorCode ERR_getErrorCode(nuint code)
{
if ((ERR_isError(code)) == 0)
{
return (ZSTD_ErrorCode)(0);
}
return (ZSTD_ErrorCode)(0 - code);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static string ERR_getErrorName(nuint code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
/*-****************************************
* Error Strings
******************************************/
public static string ERR_getErrorString(ZSTD_ErrorCode code)
{
var notErrorCode = "Unspecified error code";
switch (code)
{
case ZSTD_ErrorCode.ZSTD_error_no_error:
{
return "No error detected";
}
case ZSTD_ErrorCode.ZSTD_error_GENERIC:
{
return "Error (generic)";
}
case ZSTD_ErrorCode.ZSTD_error_prefix_unknown:
{
return "Unknown frame descriptor";
}
case ZSTD_ErrorCode.ZSTD_error_version_unsupported:
{
return "Version not supported";
}
case ZSTD_ErrorCode.ZSTD_error_frameParameter_unsupported:
{
return "Unsupported frame parameter";
}
case ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge:
{
return "Frame requires too much memory for decoding";
}
case ZSTD_ErrorCode.ZSTD_error_corruption_detected:
{
return "Corrupted block detected";
}
case ZSTD_ErrorCode.ZSTD_error_checksum_wrong:
{
return "Restored data doesn't match checksum";
}
case ZSTD_ErrorCode.ZSTD_error_parameter_unsupported:
{
return "Unsupported parameter";
}
case ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound:
{
return "Parameter is out of bound";
}
case ZSTD_ErrorCode.ZSTD_error_init_missing:
{
return "Context should be init first";
}
case ZSTD_ErrorCode.ZSTD_error_memory_allocation:
{
return "Allocation error : not enough memory";
}
case ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall:
{
return "workSpace buffer is not large enough";
}
case ZSTD_ErrorCode.ZSTD_error_stage_wrong:
{
return "Operation not authorized at current processing stage";
}
case ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge:
{
return "tableLog requires too much memory : unsupported";
}
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge:
{
return "Unsupported max Symbol Value : too large";
}
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall:
{
return "Specified maxSymbolValue is too small";
}
case ZSTD_ErrorCode.ZSTD_error_dictionary_corrupted:
{
return "Dictionary is corrupted";
}
case ZSTD_ErrorCode.ZSTD_error_dictionary_wrong:
{
return "Dictionary mismatch";
}
case ZSTD_ErrorCode.ZSTD_error_dictionaryCreation_failed:
{
return "Cannot create Dictionary from provided samples";
}
case ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall:
{
return "Destination buffer is too small";
}
case ZSTD_ErrorCode.ZSTD_error_srcSize_wrong:
{
return "Src size is incorrect";
}
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_null:
{
return "Operation on NULL destination buffer";
}
case ZSTD_ErrorCode.ZSTD_error_frameIndex_tooLarge:
{
return "Frame index is too large";
}
case ZSTD_ErrorCode.ZSTD_error_seekableIO:
{
return "An I/O error occurred when reading/seeking";
}
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_wrong:
{
return "Destination buffer is wrong";
}
case ZSTD_ErrorCode.ZSTD_error_srcBuffer_wrong:
{
return "Source buffer is wrong";
}
case ZSTD_ErrorCode.ZSTD_error_maxCode:
default:
{
return notErrorCode;
}
}
}
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-*************************************
* Acceleration
***************************************/
public partial struct FASTCOVER_accel_t
{
/* Percentage of training samples used for ZDICT_finalizeDictionary */
public uint finalize;
/* Number of dmer skipped between each dmer counted in computeFrequency */
public uint skip;
}
}

View File

@@ -0,0 +1,32 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-*************************************
* Context
***************************************/
public unsafe partial struct FASTCOVER_ctx_t
{
public byte* samples;
public nuint* offsets;
public nuint* samplesSizes;
public nuint nbSamples;
public nuint nbTrainSamples;
public nuint nbTestSamples;
public nuint nbDmers;
public uint* freqs;
public uint d;
public uint f;
public FASTCOVER_accel_t accelParams;
}
}

View File

@@ -0,0 +1,18 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* Parameters for FASTCOVER_tryParameters().
*/
public unsafe partial struct FASTCOVER_tryParameters_data_s
{
public FASTCOVER_ctx_t* ctx;
public COVER_best_s* best;
public nuint dictBufferCapacity;
public ZDICT_cover_params_t parameters;
}
}

View File

@@ -0,0 +1,22 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
Hence their body are included in next section.
*/
public unsafe partial struct FSE_CState_t
{
public nint value;
public void* stateTable;
public void* symbolTT;
public uint stateLog;
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* FSE symbol decompression API
*******************************************/
public unsafe partial struct FSE_DState_t
{
public nuint state;
/* precise table may vary, depending on U16 */
public void* table;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
/* ====== Decompression ====== */
public partial struct FSE_DTableHeader
{
public ushort tableLog;
public ushort fastMode;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct FSE_DecompressWksp
{
public fixed short ncount[256];
/* Dynamically sized */
public fixed uint dtable[1];
}
}

View File

@@ -0,0 +1,13 @@
using System;
namespace ZstdSharp.Unsafe
{
public partial struct FSE_decode_t
{
public ushort newState;
public byte symbol;
public byte nbBits;
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum FSE_repeat
{
FSE_repeat_none,
FSE_repeat_check,
FSE_repeat_valid,
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* Implementation of inlined functions
*******************************************/
public partial struct FSE_symbolCompressionTransform
{
public int deltaFindState;
public uint deltaNbBits;
}
}

View File

@@ -0,0 +1,715 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*-*************************************
* Hash Functions
***************************************/
/**
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
*/
private static nuint FASTCOVER_hashPtrToIndex(void* p, uint f, uint d)
{
if (d == 6)
{
return ZSTD_hash6Ptr(p, f);
}
return ZSTD_hash8Ptr(p, f);
}
public static FASTCOVER_accel_t* FASTCOVER_defaultAccelParameters = GetArrayPointer(new FASTCOVER_accel_t[11]
{
new FASTCOVER_accel_t
{
finalize = 100,
skip = 0,
},
new FASTCOVER_accel_t
{
finalize = 100,
skip = 0,
},
new FASTCOVER_accel_t
{
finalize = 50,
skip = 1,
},
new FASTCOVER_accel_t
{
finalize = 34,
skip = 2,
},
new FASTCOVER_accel_t
{
finalize = 25,
skip = 3,
},
new FASTCOVER_accel_t
{
finalize = 20,
skip = 4,
},
new FASTCOVER_accel_t
{
finalize = 17,
skip = 5,
},
new FASTCOVER_accel_t
{
finalize = 14,
skip = 6,
},
new FASTCOVER_accel_t
{
finalize = 13,
skip = 7,
},
new FASTCOVER_accel_t
{
finalize = 11,
skip = 8,
},
new FASTCOVER_accel_t
{
finalize = 10,
skip = 9,
},
});
/*-*************************************
* Helper functions
***************************************/
/**
* Selects the best segment in an epoch.
* Segments of are scored according to the function:
*
* Let F(d) be the frequency of all dmers with hash value d.
* Let S_i be hash value of the dmer at position i of segment S which has length k.
*
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
*
* Once the dmer with hash value d is in the dictionary we set F(d) = 0.
*/
private static COVER_segment_t FASTCOVER_selectSegment(FASTCOVER_ctx_t* ctx, uint* freqs, uint begin, uint end, ZDICT_cover_params_t parameters, ushort* segmentFreqs)
{
uint k = parameters.k;
uint d = parameters.d;
uint f = ctx->f;
uint dmersInK = k - d + 1;
COVER_segment_t bestSegment = new COVER_segment_t
{
begin = 0,
end = 0,
score = 0,
};
COVER_segment_t activeSegment;
activeSegment.begin = begin;
activeSegment.end = begin;
activeSegment.score = 0;
while (activeSegment.end < end)
{
nuint idx = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.end), f, d);
if (segmentFreqs[idx] == 0)
{
activeSegment.score += freqs[idx];
}
activeSegment.end += 1;
segmentFreqs[idx] += 1;
if (activeSegment.end - activeSegment.begin == dmersInK + 1)
{
nuint delIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.begin), f, d);
segmentFreqs[delIndex] -= 1;
if (segmentFreqs[delIndex] == 0)
{
activeSegment.score -= freqs[delIndex];
}
activeSegment.begin += 1;
}
if (activeSegment.score > bestSegment.score)
{
bestSegment = activeSegment;
}
}
while (activeSegment.begin < end)
{
nuint delIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.begin), f, d);
segmentFreqs[delIndex] -= 1;
activeSegment.begin += 1;
}
{
uint pos;
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos)
{
nuint i = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + pos), f, d);
freqs[i] = 0;
}
}
return bestSegment;
}
private static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters, nuint maxDictSize, uint f, uint accel)
{
if (parameters.d == 0 || parameters.k == 0)
{
return 0;
}
if (parameters.d != 6 && parameters.d != 8)
{
return 0;
}
if (parameters.k > maxDictSize)
{
return 0;
}
if (parameters.d > parameters.k)
{
return 0;
}
if (f > 31 || f == 0)
{
return 0;
}
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1)
{
return 0;
}
if (accel > 10 || accel == 0)
{
return 0;
}
return 1;
}
/**
* Clean up a context initialized with `FASTCOVER_ctx_init()`.
*/
private static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx)
{
if (ctx == null)
{
return;
}
free((void*)ctx->freqs);
ctx->freqs = null;
free((void*)ctx->offsets);
ctx->offsets = null;
}
/**
* Calculate for frequency of hash value of each dmer in ctx->samples
*/
private static void FASTCOVER_computeFrequency(uint* freqs, FASTCOVER_ctx_t* ctx)
{
uint f = ctx->f;
uint d = ctx->d;
uint skip = ctx->accelParams.skip;
uint readLength = ((d) > (8) ? (d) : (8));
nuint i;
assert(ctx->nbTrainSamples >= 5);
assert(ctx->nbTrainSamples <= ctx->nbSamples);
for (i = 0; i < ctx->nbTrainSamples; i++)
{
nuint start = ctx->offsets[i];
nuint currSampleEnd = ctx->offsets[i + 1];
while (start + readLength <= currSampleEnd)
{
nuint dmerIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + start), f, d);
freqs[dmerIndex]++;
start = start + skip + 1;
}
}
}
/**
* Prepare a context for dictionary building.
* The context is only dependent on the parameter `d` and can used multiple
* times.
* Returns 0 on success or error code on error.
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
*/
private static nuint FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, uint d, double splitPoint, uint f, FASTCOVER_accel_t accelParams)
{
byte* samples = (byte*)(samplesBuffer);
nuint totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
uint nbTrainSamples = splitPoint < 1.0 ? (uint)((double)(nbSamples) * splitPoint) : nbSamples;
uint nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
nuint trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
nuint testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
if (totalSamplesSize < ((d) > ((nuint)(sizeof(ulong))) ? (d) : ((nuint)(sizeof(ulong)))) || totalSamplesSize >= (nuint)((nuint)(sizeof(nuint)) == 8 ? (unchecked((uint)(-1))) : ((uint)(1) * (1U << 30))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (nbTrainSamples < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (nbTestSamples < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
memset((void*)ctx, 0, (nuint)(sizeof(FASTCOVER_ctx_t)));
ctx->samples = samples;
ctx->samplesSizes = samplesSizes;
ctx->nbSamples = nbSamples;
ctx->nbTrainSamples = nbTrainSamples;
ctx->nbTestSamples = nbTestSamples;
ctx->nbDmers = trainingSamplesSize - ((d) > ((nuint)(sizeof(ulong))) ? (d) : ((nuint)(sizeof(ulong)))) + 1;
ctx->d = d;
ctx->f = f;
ctx->accelParams = accelParams;
ctx->offsets = (nuint*)(calloc((nbSamples + 1), (nuint)(sizeof(nuint))));
if (ctx->offsets == null)
{
FASTCOVER_ctx_destroy(ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
{
uint i;
ctx->offsets[0] = 0;
assert(nbSamples >= 5);
for (i = 1; i <= nbSamples; ++i)
{
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
}
}
ctx->freqs = (uint*)(calloc((nuint)((ulong)(1) << (int)f), (nuint)(sizeof(uint))));
if (ctx->freqs == null)
{
FASTCOVER_ctx_destroy(ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
FASTCOVER_computeFrequency(ctx->freqs, ctx);
return 0;
}
/**
* Given the prepared context build the dictionary.
*/
private static nuint FASTCOVER_buildDictionary(FASTCOVER_ctx_t* ctx, uint* freqs, void* dictBuffer, nuint dictBufferCapacity, ZDICT_cover_params_t parameters, ushort* segmentFreqs)
{
byte* dict = (byte*)(dictBuffer);
nuint tail = dictBufferCapacity;
COVER_epoch_info_t epochs = COVER_computeEpochs((uint)(dictBufferCapacity), (uint)(ctx->nbDmers), parameters.k, 1);
nuint maxZeroScoreRun = 10;
nuint zeroScoreRun = 0;
nuint epoch;
for (epoch = 0; tail > 0; epoch = (nuint)((epoch + 1) % epochs.num))
{
uint epochBegin = (uint)(epoch * epochs.size);
uint epochEnd = epochBegin + epochs.size;
nuint segmentSize;
COVER_segment_t segment = FASTCOVER_selectSegment(ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
if (segment.score == 0)
{
if (++zeroScoreRun >= maxZeroScoreRun)
{
break;
}
continue;
}
zeroScoreRun = 0;
segmentSize = ((segment.end - segment.begin + parameters.d - 1) < (tail) ? (segment.end - segment.begin + parameters.d - 1) : (tail));
if (segmentSize < parameters.d)
{
break;
}
tail -= segmentSize;
memcpy((void*)(dict + tail), (void*)(ctx->samples + segment.begin), segmentSize);
}
return tail;
}
/**
* Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading.
*/
private static void FASTCOVER_tryParameters(void* opaque)
{
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)(opaque);
FASTCOVER_ctx_t* ctx = data->ctx;
ZDICT_cover_params_t parameters = data->parameters;
nuint dictBufferCapacity = data->dictBufferCapacity;
nuint totalCompressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
ushort* segmentFreqs = (ushort*)(calloc((nuint)((ulong)(1) << (int)ctx->f), (nuint)(2)));
byte* dict = (byte*)(malloc(dictBufferCapacity));
COVER_dictSelection selection = COVER_dictSelectionError((unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))));
uint* freqs = (uint*)(malloc((nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(4))));
if (segmentFreqs == null || dict == null || freqs == null)
{
goto _cleanup;
}
memcpy((void*)freqs, (void*)ctx->freqs, (nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(sizeof(uint))));
{
nuint tail = FASTCOVER_buildDictionary(ctx, freqs, (void*)dict, dictBufferCapacity, parameters, segmentFreqs);
uint nbFinalizeSamples = (uint)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize);
if ((COVER_dictSelectionIsError(selection)) != 0)
{
goto _cleanup;
}
}
_cleanup:
free((void*)dict);
COVER_best_finish(data->best, parameters, selection);
free((void*)data);
free((void*)segmentFreqs);
COVER_dictSelectionFree(selection);
free((void*)freqs);
}
private static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, ZDICT_cover_params_t* coverParams)
{
coverParams->k = fastCoverParams.k;
coverParams->d = fastCoverParams.d;
coverParams->steps = fastCoverParams.steps;
coverParams->nbThreads = fastCoverParams.nbThreads;
coverParams->splitPoint = fastCoverParams.splitPoint;
coverParams->zParams = fastCoverParams.zParams;
coverParams->shrinkDict = fastCoverParams.shrinkDict;
}
private static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, ZDICT_fastCover_params_t* fastCoverParams, uint f, uint accel)
{
fastCoverParams->k = coverParams.k;
fastCoverParams->d = coverParams.d;
fastCoverParams->steps = coverParams.steps;
fastCoverParams->nbThreads = coverParams.nbThreads;
fastCoverParams->splitPoint = coverParams.splitPoint;
fastCoverParams->f = f;
fastCoverParams->accel = accel;
fastCoverParams->zParams = coverParams.zParams;
fastCoverParams->shrinkDict = coverParams.shrinkDict;
}
/*! ZDICT_trainFromBuffer_fastCover():
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
* d and k are required.
* All other parameters are optional, will use default values if not provided
* The resulting dictionary will be saved into `dictBuffer`.
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/
public static nuint ZDICT_trainFromBuffer_fastCover(void* dictBuffer, nuint dictBufferCapacity, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, ZDICT_fastCover_params_t parameters)
{
byte* dict = (byte*)(dictBuffer);
FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
g_displayLevel = (int)parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0;
parameters.f = (uint)(parameters.f == 0 ? 20 : parameters.f);
parameters.accel = (uint)(parameters.accel == 0 ? 1 : parameters.accel);
memset((void*)&coverParams, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
FASTCOVER_convertToCoverParams(parameters, &coverParams);
if ((FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f, parameters.accel)) == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (nbSamples == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (dictBufferCapacity < 256)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
{
nuint initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, coverParams.d, parameters.splitPoint, parameters.f, accelParams);
if ((ERR_isError(initVal)) != 0)
{
return initVal;
}
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
{
ushort* segmentFreqs = (ushort*)(calloc((nuint)((ulong)(1) << (int)parameters.f), (nuint)(2)));
nuint tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, dictBufferCapacity, coverParams, segmentFreqs);
uint nbFinalizeSamples = (uint)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
nuint dictionarySize = ZDICT_finalizeDictionary((void*)dict, dictBufferCapacity, (void*)(dict + tail), dictBufferCapacity - tail, samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
if ((ERR_isError(dictionarySize)) == 0)
{
;
}
FASTCOVER_ctx_destroy(&ctx);
free((void*)segmentFreqs);
return dictionarySize;
}
}
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
* The same requirements as above hold for all the parameters except `parameters`.
* This function tries many parameter combinations (specifically, k and d combinations)
* and picks the best parameters. `*parameters` is filled with the best parameters found,
* dictionary constructed with those parameters is stored in `dictBuffer`.
* All of the parameters d, k, steps, f, and accel are optional.
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
* if steps is zero it defaults to its default value.
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
* If f is zero, default value of 20 is used.
* If accel is zero, default value of 1 is used.
*
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* On success `*parameters` contains the parameters selected.
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
*/
public static nuint ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, nuint dictBufferCapacity, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, ZDICT_fastCover_params_t* parameters)
{
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
uint nbThreads = parameters->nbThreads;
double splitPoint = parameters->splitPoint <= 0.0 ? 0.75 : parameters->splitPoint;
uint kMinD = (uint)(parameters->d == 0 ? 6 : parameters->d);
uint kMaxD = (uint)(parameters->d == 0 ? 8 : parameters->d);
uint kMinK = (uint)(parameters->k == 0 ? 50 : parameters->k);
uint kMaxK = (uint)(parameters->k == 0 ? 2000 : parameters->k);
uint kSteps = (uint)(parameters->steps == 0 ? 40 : parameters->steps);
uint kStepSize = (((kMaxK - kMinK) / kSteps) > (1) ? ((kMaxK - kMinK) / kSteps) : (1));
uint kIterations = (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
uint f = (uint)(parameters->f == 0 ? 20 : parameters->f);
uint accel = (uint)(parameters->accel == 0 ? 1 : parameters->accel);
uint shrinkDict = 0;
int displayLevel = (int)parameters->zParams.notificationLevel;
uint iteration = 1;
uint d;
uint k;
COVER_best_s best;
int warned = 0;
if (splitPoint <= 0 || splitPoint > 1)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (accel == 0 || accel > 10)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (kMinK < kMaxD || kMaxK < kMinK)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (nbSamples == 0)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (dictBufferCapacity < 256)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
if (nbThreads > 1)
{
throw new NotImplementedException("Multiple threads are not supported");
}
COVER_best_init(&best);
memset((void*)&coverParams, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel];
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
if (displayLevel >= 2)
{
;
}
for (d = kMinD; d <= kMaxD; d += 2)
{
FASTCOVER_ctx_t ctx;
if (displayLevel >= 3)
{
;
}
{
nuint initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams);
if ((ERR_isError(initVal)) != 0)
{
if (displayLevel >= 1)
{
;
}
COVER_best_destroy(&best);
return initVal;
}
}
if (warned == 0)
{
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
warned = 1;
}
for (k = kMinK; k <= kMaxK; k += kStepSize)
{
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)(malloc((nuint)(sizeof(FASTCOVER_tryParameters_data_s))));
if (displayLevel >= 3)
{
;
}
if (data == null)
{
if (displayLevel >= 1)
{
;
}
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
data->ctx = &ctx;
data->best = &best;
data->dictBufferCapacity = dictBufferCapacity;
data->parameters = coverParams;
data->parameters.k = k;
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (uint)g_displayLevel;
if ((FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) == 0)
{
free((void*)data);
continue;
}
COVER_best_start(&best);
FASTCOVER_tryParameters((void*)data);
++iteration;
}
COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx);
}
if (displayLevel >= 2)
{
;
}
{
nuint dictSize = best.dictSize;
if ((ERR_isError(best.compressedSize)) != 0)
{
nuint compressedSize = best.compressedSize;
COVER_best_destroy(&best);
return compressedSize;
}
FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
memcpy(dictBuffer, best.dict, dictSize);
COVER_best_destroy(&best);
return dictSize;
}
}
}
}

View File

@@ -0,0 +1,158 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initCState(FSE_CState_t* statePtr, uint* ct)
{
void* ptr = (void*)ct;
ushort* u16ptr = (ushort*)(ptr);
uint tableLog = MEM_read16(ptr);
statePtr->value = (nint)(1) << (int)tableLog;
statePtr->stateTable = u16ptr + 2;
statePtr->symbolTT = ct + 1 + (tableLog != 0 ? (1 << (int)(tableLog - 1)) : 1);
statePtr->stateLog = tableLog;
}
/*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initCState2(FSE_CState_t* statePtr, uint* ct, uint symbol)
{
FSE_initCState(statePtr, ct);
{
FSE_symbolCompressionTransform symbolTT = ((FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
ushort* stateTable = (ushort*)(statePtr->stateTable);
uint nbBitsOut = (uint)((symbolTT.deltaNbBits + (uint)((1 << 15))) >> 16);
statePtr->value = (nint)((nbBitsOut << 16) - symbolTT.deltaNbBits);
statePtr->value = (nint)(stateTable[(statePtr->value >> (int)nbBitsOut) + symbolTT.deltaFindState]);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, uint symbol)
{
FSE_symbolCompressionTransform symbolTT = ((FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
ushort* stateTable = (ushort*)(statePtr->stateTable);
uint nbBitsOut = (uint)(((nuint)statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, (nuint)statePtr->value, nbBitsOut);
statePtr->value = (nint)(stateTable[(statePtr->value >> (int)nbBitsOut) + symbolTT.deltaFindState]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_flushCState(BIT_CStream_t* bitC, FSE_CState_t* statePtr)
{
BIT_addBits(bitC, (nuint)statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/* FSE_getMaxNbBits() :
* Approximate maximum cost of a symbol, in bits.
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_getMaxNbBits(void* symbolTTPtr, uint symbolValue)
{
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(symbolTTPtr);
return (symbolTT[symbolValue].deltaNbBits + (uint)(((1 << 16) - 1))) >> 16;
}
/* FSE_bitCost() :
* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_bitCost(void* symbolTTPtr, uint tableLog, uint symbolValue, uint accuracyLog)
{
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(symbolTTPtr);
uint minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
uint threshold = (minNbBits + 1) << 16;
assert(tableLog < 16);
assert(accuracyLog < 31 - tableLog);
{
uint tableSize = (uint)(1 << (int)tableLog);
uint deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
uint normalizedDeltaFromThreshold = (deltaFromThreshold << (int)accuracyLog) >> (int)tableLog;
uint bitMultiplier = (uint)(1 << (int)accuracyLog);
assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
assert(normalizedDeltaFromThreshold <= bitMultiplier);
return (minNbBits + 1) * bitMultiplier - normalizedDeltaFromThreshold;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, uint* dt)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_peekSymbol(FSE_DState_t* DStatePtr)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
nuint lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
byte symbol = DInfo.symbol;
nuint lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
byte symbol = DInfo.symbol;
nuint lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_endOfDState(FSE_DState_t* DStatePtr)
{
return ((DStatePtr->state == 0) ? 1U : 0U);
}
}
}

View File

@@ -0,0 +1,960 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
*/
public static nuint FSE_buildCTable_wksp(uint* ct, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
uint tableSize = (uint)(1 << (int)tableLog);
uint tableMask = tableSize - 1;
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCT = (void*)(((uint*)(ptr)) + 1 + (tableLog != 0 ? tableSize >> 1 : 1));
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCT);
uint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
uint* cumul = (uint*)(workSpace);
byte* tableSymbol = (byte*)(cumul + (maxSymbolValue + 2));
uint highThreshold = tableSize - 1;
if (((nuint)(workSpace) & 3) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (((nuint)(sizeof(uint)) * (maxSymbolValue + 2 + (1UL << (int)(tableLog - 2)))) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
tableU16[-2] = (ushort)(tableLog);
tableU16[-1] = (ushort)(maxSymbolValue);
assert(tableLog < 16);
{
uint u;
cumul[0] = 0;
for (u = 1; u <= maxSymbolValue + 1; u++)
{
if (normalizedCounter[u - 1] == -1)
{
cumul[u] = cumul[u - 1] + 1;
tableSymbol[highThreshold--] = (byte)(u - 1);
}
else
{
cumul[u] = cumul[u - 1] + (ushort)(normalizedCounter[u - 1]);
}
}
cumul[maxSymbolValue + 1] = tableSize + 1;
}
{
uint position = 0;
uint symbol;
for (symbol = 0; symbol <= maxSymbolValue; symbol++)
{
int nbOccurrences;
int freq = normalizedCounter[symbol];
for (nbOccurrences = 0; nbOccurrences < freq; nbOccurrences++)
{
tableSymbol[position] = (byte)(symbol);
position = (position + step) & tableMask;
while (position > highThreshold)
{
position = (position + step) & tableMask;
}
}
}
assert(position == 0);
}
{
uint u;
for (u = 0; u < tableSize; u++)
{
byte s = tableSymbol[u];
tableU16[cumul[s]++] = (ushort)(tableSize + u);
}
}
{
uint total = 0;
uint s;
for (s = 0; s <= maxSymbolValue; s++)
{
switch (normalizedCounter[s])
{
case 0:
{
symbolTT[s].deltaNbBits = ((tableLog + 1) << 16) - (uint)((1 << (int)tableLog));
}
break;
case -1:
case 1:
{
symbolTT[s].deltaNbBits = (tableLog << 16) - (uint)((1 << (int)tableLog));
}
symbolTT[s].deltaFindState = (int)(total - 1);
total++;
break;
default:
{
uint maxBitsOut = tableLog - BIT_highbit32((uint)(normalizedCounter[s] - 1));
uint minStatePlus = (uint)(normalizedCounter[s] << (int)maxBitsOut);
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = (int)(total - (ushort)(normalizedCounter[s]));
total += (uint)(normalizedCounter[s]);
}
break;
}
}
}
return 0;
}
/*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_buildCTable(uint* ct, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
byte* tableSymbol = stackalloc byte[4096];
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, (void*)tableSymbol, (nuint)(sizeof(byte) * 4096));
}
/*-**************************************************************
* FSE NCount encoding
****************************************************************/
public static nuint FSE_NCountWriteBound(uint maxSymbolValue, uint tableLog)
{
nuint maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
return maxSymbolValue != 0 ? maxHeaderSize : 512;
}
private static nuint FSE_writeNCount_generic(void* header, nuint headerBufferSize, short* normalizedCounter, uint maxSymbolValue, uint tableLog, uint writeIsSafe)
{
byte* ostart = (byte*)(header);
byte* @out = ostart;
byte* oend = ostart + headerBufferSize;
int nbBits;
int tableSize = 1 << (int)tableLog;
int remaining;
int threshold;
uint bitStream = 0;
int bitCount = 0;
uint symbol = 0;
uint alphabetSize = maxSymbolValue + 1;
int previousIs0 = 0;
bitStream += (tableLog - 5) << bitCount;
bitCount += 4;
remaining = tableSize + 1;
threshold = tableSize;
nbBits = (int)(tableLog + 1);
while ((symbol < alphabetSize) && (remaining > 1))
{
if (previousIs0 != 0)
{
uint start = symbol;
while ((symbol < alphabetSize) && (normalizedCounter[symbol]) == 0)
{
symbol++;
}
if (symbol == alphabetSize)
{
break;
}
while (symbol >= start + 24)
{
start += 24;
bitStream += 0xFFFFU << bitCount;
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
}
while (symbol >= start + 3)
{
start += 3;
bitStream += (uint)(3 << bitCount);
bitCount += 2;
}
bitStream += (symbol - start) << bitCount;
bitCount += 2;
if (bitCount > 16)
{
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
bitCount -= 16;
}
}
{
int count = normalizedCounter[symbol++];
int max = (2 * threshold - 1) - remaining;
remaining -= count < 0 ? -count : count;
count++;
if (count >= threshold)
{
count += max;
}
bitStream += (uint)(count << bitCount);
bitCount += nbBits;
bitCount -= ((count < max) ? 1 : 0);
previousIs0 = ((count == 1) ? 1 : 0);
if (remaining < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
while (remaining < threshold)
{
nbBits--;
threshold >>= 1;
}
}
if (bitCount > 16)
{
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
bitCount -= 16;
}
}
if (remaining != 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
assert(symbol <= alphabetSize);
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += (bitCount + 7) / 8;
return (nuint)((@out - ostart));
}
/*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */
public static nuint FSE_writeNCount(void* buffer, nuint bufferSize, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (tableLog < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
{
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
}
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
/*-**************************************************************
* FSE Compression Code
****************************************************************/
public static uint* FSE_createCTable(uint maxSymbolValue, uint tableLog)
{
nuint size;
if (tableLog > 15)
{
tableLog = 15;
}
size = ((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2)) * (nuint)(sizeof(uint));
return (uint*)(malloc(size));
}
public static void FSE_freeCTable(uint* ct)
{
free((void*)(ct));
}
/* provides the minimum logSize to safely represent a distribution */
[InlineMethod.Inline]
private static uint FSE_minTableLog(nuint srcSize, uint maxSymbolValue)
{
uint minBitsSrc = BIT_highbit32((uint)(srcSize)) + 1;
uint minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
uint minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
assert(srcSize > 1);
return minBits;
}
/* *****************************************
* FSE advanced API
***************************************** */
public static uint FSE_optimalTableLog_internal(uint maxTableLog, nuint srcSize, uint maxSymbolValue, uint minus)
{
uint maxBitsSrc = BIT_highbit32((uint)(srcSize - 1)) - minus;
uint tableLog = maxTableLog;
uint minBits = FSE_minTableLog(srcSize, maxSymbolValue);
assert(srcSize > 1);
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
if (maxBitsSrc < tableLog)
{
tableLog = maxBitsSrc;
}
if (minBits > tableLog)
{
tableLog = minBits;
}
if (tableLog < 5)
{
tableLog = 5;
}
if (tableLog > (uint)((14 - 2)))
{
tableLog = (uint)((14 - 2));
}
return tableLog;
}
/*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */
public static uint FSE_optimalTableLog(uint maxTableLog, nuint srcSize, uint maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}
/* Secondary normalization method.
To be used when primary method fails. */
private static nuint FSE_normalizeM2(short* norm, uint tableLog, uint* count, nuint total, uint maxSymbolValue, short lowProbCount)
{
short NOT_YET_ASSIGNED = (short)-2;
uint s;
uint distributed = 0;
uint ToDistribute;
uint lowThreshold = (uint)(total >> (int)tableLog);
uint lowOne = (uint)((total * 3) >> (int)(tableLog + 1));
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] == 0)
{
norm[s] = 0;
continue;
}
if (count[s] <= lowThreshold)
{
norm[s] = lowProbCount;
distributed++;
total -= count[s];
continue;
}
if (count[s] <= lowOne)
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
norm[s] = NOT_YET_ASSIGNED;
}
ToDistribute = (uint)((1 << (int)tableLog)) - distributed;
if (ToDistribute == 0)
{
return 0;
}
if ((total / ToDistribute) > lowOne)
{
lowOne = (uint)((total * 3) / (ToDistribute * 2));
for (s = 0; s <= maxSymbolValue; s++)
{
if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne))
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
}
ToDistribute = (uint)((1 << (int)tableLog)) - distributed;
}
if (distributed == maxSymbolValue + 1)
{
uint maxV = 0, maxC = 0;
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] > maxC)
{
maxV = s;
maxC = count[s];
}
}
norm[maxV] += (short)(short)(ToDistribute);
return 0;
}
if (total == 0)
{
for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
{
if (norm[s] > 0)
{
ToDistribute--;
norm[s]++;
}
}
return 0;
}
{
ulong vStepLog = 62 - tableLog;
ulong mid = (1UL << (int)(vStepLog - 1)) - 1;
ulong rStep = (((((ulong)(1) << (int)vStepLog) * ToDistribute) + mid) / ((uint)(total)));
ulong tmpTotal = mid;
for (s = 0; s <= maxSymbolValue; s++)
{
if (norm[s] == NOT_YET_ASSIGNED)
{
ulong end = tmpTotal + (count[s] * rStep);
uint sStart = (uint)(tmpTotal >> (int)vStepLog);
uint sEnd = (uint)(end >> (int)vStepLog);
uint weight = sEnd - sStart;
if (weight < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
norm[s] = (short)(weight);
tmpTotal = end;
}
}
}
return 0;
}
/*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
useLowProbCount is a boolean parameter which trades off compressed size for
faster header decoding. When it is set to 1, the compressed data will be slightly
smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
is a good default, since header deserialization makes a big speed difference.
Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
@return : tableLog,
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_normalizeCount(short* normalizedCounter, uint tableLog, uint* count, nuint total, uint maxSymbolValue, uint useLowProbCount)
{
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
if (tableLog < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (tableLog < FSE_minTableLog(total, maxSymbolValue))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
{
short lowProbCount = (short)(useLowProbCount != 0 ? -1 : 1);
ulong scale = 62 - tableLog;
ulong step = (((ulong)(1) << 62) / ((uint)(total)));
ulong vStep = 1UL << (int)(scale - 20);
int stillToDistribute = 1 << (int)tableLog;
uint s;
uint largest = 0;
short largestP = 0;
uint lowThreshold = (uint)(total >> (int)tableLog);
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] == total)
{
return 0;
}
if (count[s] == 0)
{
normalizedCounter[s] = 0;
continue;
}
if (count[s] <= lowThreshold)
{
normalizedCounter[s] = lowProbCount;
stillToDistribute--;
}
else
{
short proba = (short)((count[s] * step) >> (int)scale);
if (proba < 8)
{
ulong restToBeat = vStep * rtbTable[proba];
proba += (short)((((count[s] * step) - ((ulong)(proba) << (int)scale) > restToBeat) ? 1 : 0));
}
if (proba > largestP)
{
largestP = proba;
largest = s;
}
normalizedCounter[s] = proba;
stillToDistribute -= proba;
}
}
if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
{
nuint errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
if ((ERR_isError(errorCode)) != 0)
{
return errorCode;
}
}
else
{
normalizedCounter[largest] += (short)(short)(stillToDistribute);
}
}
return tableLog;
}
/* fake FSE_CTable, for raw (uncompressed) input */
public static nuint FSE_buildCTable_raw(uint* ct, uint nbBits)
{
uint tableSize = (uint)(1 << (int)nbBits);
uint tableMask = tableSize - 1;
uint maxSymbolValue = tableMask;
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCT = (void*)(((uint*)(ptr)) + 1 + (tableSize >> 1));
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCT);
uint s;
if (nbBits < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
tableU16[-2] = (ushort)(nbBits);
tableU16[-1] = (ushort)(maxSymbolValue);
for (s = 0; s < tableSize; s++)
{
tableU16[s] = (ushort)(tableSize + s);
}
{
uint deltaNbBits = (nbBits << 16) - (uint)((1 << (int)nbBits));
for (s = 0; s <= maxSymbolValue; s++)
{
symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = (int)(s - 1);
}
}
return 0;
}
/* fake FSE_CTable, for rle input (always same symbol) */
public static nuint FSE_buildCTable_rle(uint* ct, byte symbolValue)
{
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCTptr = (void*)((uint*)(ptr) + 2);
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCTptr);
tableU16[-2] = (ushort)(0);
tableU16[-1] = (ushort)(symbolValue);
tableU16[0] = 0;
tableU16[1] = 0;
symbolTT[symbolValue].deltaNbBits = 0;
symbolTT[symbolValue].deltaFindState = 0;
return 0;
}
private static nuint FSE_compress_usingCTable_generic(void* dst, nuint dstSize, void* src, nuint srcSize, uint* ct, uint fast)
{
byte* istart = (byte*)(src);
byte* iend = istart + srcSize;
byte* ip = iend;
BIT_CStream_t bitC;
FSE_CState_t CState1, CState2;
if (srcSize <= 2)
{
return 0;
}
{
nuint initError = BIT_initCStream(&bitC, dst, dstSize);
if ((ERR_isError(initError)) != 0)
{
return 0;
}
}
if ((srcSize & 1) != 0)
{
FSE_initCState2(&CState1, ct, *--ip);
FSE_initCState2(&CState2, ct, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
else
{
FSE_initCState2(&CState2, ct, *--ip);
FSE_initCState2(&CState1, ct, *--ip);
}
srcSize -= 2;
if (((nuint)(sizeof(nuint)) * 8 > (uint)((14 - 2) * 4 + 7)) && (srcSize & 2) != 0)
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
while (ip > istart)
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if ((nuint)(sizeof(nuint)) * 8 < (uint)((14 - 2) * 2 + 7))
{
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if ((nuint)(sizeof(nuint)) * 8 > (uint)((14 - 2) * 4 + 7))
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
FSE_flushCState(&bitC, &CState2);
FSE_flushCState(&bitC, &CState1);
return BIT_closeCStream(&bitC);
}
/*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_compress_usingCTable(void* dst, nuint dstSize, void* src, nuint srcSize, uint* ct)
{
uint fast = (((dstSize >= ((srcSize) + ((srcSize) >> 7) + 4 + (nuint)(sizeof(nuint))))) ? 1U : 0U);
if (fast != 0)
{
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
}
else
{
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}
}
/*-*****************************************
* Tool functions
******************************************/
public static nuint FSE_compressBound(nuint size)
{
return (512 + ((size) + ((size) >> 7) + 4 + (nuint)(sizeof(nuint))));
}
/* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` size must be `(1<<tableLog)`.
*/
public static nuint FSE_compress_wksp(void* dst, nuint dstSize, void* src, nuint srcSize, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
byte* ostart = (byte*)(dst);
byte* op = ostart;
byte* oend = ostart + dstSize;
uint* count = stackalloc uint[256];
short* norm = stackalloc short[256];
uint* CTable = (uint*)(workSpace);
nuint CTableSize = ((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2));
void* scratchBuffer = (void*)(CTable + CTableSize);
nuint scratchBufferSize = wkspSize - (CTableSize * (nuint)(4));
if (wkspSize < (((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2)) + (uint)(((tableLog > 12) ? (1 << (int)(tableLog - 2)) : 1024))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (srcSize <= 1)
{
return 0;
}
if (maxSymbolValue == 0)
{
maxSymbolValue = 255;
}
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
{
nuint maxCount = HIST_count_wksp((uint*)count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize);
if ((ERR_isError(maxCount)) != 0)
{
return maxCount;
}
if (maxCount == srcSize)
{
return 1;
}
if (maxCount == 1)
{
return 0;
}
if (maxCount < (srcSize >> 7))
{
return 0;
}
}
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
{
nuint _var_err__ = FSE_normalizeCount((short*)norm, tableLog, (uint*)count, srcSize, maxSymbolValue, ((srcSize >= 2048) ? 1U : 0U));
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
nuint nc_err = FSE_writeNCount((void*)op, (nuint)(oend - op), (short*)norm, maxSymbolValue, tableLog);
if ((ERR_isError(nc_err)) != 0)
{
return nc_err;
}
op += nc_err;
}
{
nuint _var_err__ = FSE_buildCTable_wksp(CTable, (short*)norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
nuint cSize = FSE_compress_usingCTable((void*)op, (nuint)(oend - op), src, srcSize, CTable);
if ((ERR_isError(cSize)) != 0)
{
return cSize;
}
if (cSize == 0)
{
return 0;
}
op += cSize;
}
if ((nuint)(op - ostart) >= srcSize - 1)
{
return 0;
}
return (nuint)(op - ostart);
}
/*-*****************************************
* FSE advanced functions
******************************************/
/*! FSE_compress2() :
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
*/
public static nuint FSE_compress2(void* dst, nuint dstCapacity, void* src, nuint srcSize, uint maxSymbolValue, uint tableLog)
{
fseWkspMax_t scratchBuffer;
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, (void*)&scratchBuffer, (nuint)(sizeof(fseWkspMax_t)));
}
/*-****************************************
* FSE simple functions
******************************************/
/*! FSE_compress() :
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
@return : size of compressed data (<= dstCapacity).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/
public static nuint FSE_compress(void* dst, nuint dstCapacity, void* src, nuint srcSize)
{
return FSE_compress2(dst, dstCapacity, src, srcSize, 255, (uint)((13 - 2)));
}
}
}

View File

@@ -0,0 +1,446 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/* Function templates */
public static uint* FSE_createDTable(uint tableLog)
{
if (tableLog > 15)
{
tableLog = 15;
}
return (uint*)(malloc((uint)((1 + (1 << (int)(tableLog)))) * (nuint)(sizeof(uint))));
}
public static void FSE_freeDTable(uint* dt)
{
free((void*)(dt));
}
private static nuint FSE_buildDTable_internal(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
void* tdPtr = (void*)(dt + 1);
FSE_decode_t* tableDecode = (FSE_decode_t*)(tdPtr);
ushort* symbolNext = (ushort*)(workSpace);
byte* spread = (byte*)(symbolNext + maxSymbolValue + 1);
uint maxSV1 = maxSymbolValue + 1;
uint tableSize = (uint)(1 << (int)tableLog);
uint highThreshold = tableSize - 1;
if (((nuint)(sizeof(short)) * (maxSymbolValue + 1) + (1UL << (int)tableLog) + 8) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge)));
}
if (maxSymbolValue > 255)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge)));
}
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
{
FSE_DTableHeader DTableH;
DTableH.tableLog = (ushort)(tableLog);
DTableH.fastMode = 1;
{
short largeLimit = (short)(1 << (int)(tableLog - 1));
uint s;
for (s = 0; s < maxSV1; s++)
{
if (normalizedCounter[s] == -1)
{
tableDecode[highThreshold--].symbol = (byte)(s);
symbolNext[s] = 1;
}
else
{
if (normalizedCounter[s] >= largeLimit)
{
DTableH.fastMode = 0;
}
symbolNext[s] = (ushort)(normalizedCounter[s]);
}
}
}
memcpy((void*)(dt), (void*)(&DTableH), ((nuint)(sizeof(FSE_DTableHeader))));
}
if (highThreshold == tableSize - 1)
{
nuint tableMask = tableSize - 1;
nuint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
{
ulong add = 0x0101010101010101UL;
nuint pos = 0;
ulong sv = 0;
uint s;
for (s = 0; s < maxSV1; ++s , sv += add)
{
int i;
int n = normalizedCounter[s];
MEM_write64((void*)(spread + pos), sv);
for (i = 8; i < n; i += 8)
{
MEM_write64((void*)(spread + pos + i), sv);
}
pos += (nuint)n;
}
}
{
nuint position = 0;
nuint s;
nuint unroll = 2;
assert(tableSize % unroll == 0);
for (s = 0; s < (nuint)(tableSize); s += unroll)
{
nuint u;
for (u = 0; u < unroll; ++u)
{
nuint uPosition = (position + (u * step)) & tableMask;
tableDecode[uPosition].symbol = spread[s + u];
}
position = (position + (unroll * step)) & tableMask;
}
assert(position == 0);
}
}
else
{
uint tableMask = tableSize - 1;
uint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
uint s, position = 0;
for (s = 0; s < maxSV1; s++)
{
int i;
for (i = 0; i < normalizedCounter[s]; i++)
{
tableDecode[position].symbol = (byte)(s);
position = (position + step) & tableMask;
while (position > highThreshold)
{
position = (position + step) & tableMask;
}
}
}
if (position != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
}
{
uint u;
for (u = 0; u < tableSize; u++)
{
byte symbol = (byte)(tableDecode[u].symbol);
uint nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (byte)(tableLog - BIT_highbit32(nextState));
tableDecode[u].newState = (ushort)((nextState << (int)(tableDecode[u].nbBits)) - tableSize);
}
}
return 0;
}
public static nuint FSE_buildDTable_wksp(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
}
/*-*******************************************************
* Decompression (Byte symbols)
*********************************************************/
public static nuint FSE_buildDTable_rle(uint* dt, byte symbolValue)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
void* dPtr = (void*)(dt + 1);
FSE_decode_t* cell = (FSE_decode_t*)(dPtr);
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
cell->nbBits = 0;
return 0;
}
public static nuint FSE_buildDTable_raw(uint* dt, uint nbBits)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
void* dPtr = (void*)(dt + 1);
FSE_decode_t* dinfo = (FSE_decode_t*)(dPtr);
uint tableSize = (uint)(1 << (int)nbBits);
uint tableMask = tableSize - 1;
uint maxSV1 = tableMask + 1;
uint s;
if (nbBits < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
DTableH->tableLog = (ushort)(nbBits);
DTableH->fastMode = 1;
for (s = 0; s < maxSV1; s++)
{
dinfo[s].newState = 0;
dinfo[s].symbol = (byte)(s);
dinfo[s].nbBits = (byte)(nbBits);
}
return 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_decompress_usingDTable_generic(void* dst, nuint maxDstSize, void* cSrc, nuint cSrcSize, uint* dt, uint fast)
{
byte* ostart = (byte*)(dst);
byte* op = ostart;
byte* omax = op + maxDstSize;
byte* olimit = omax - 3;
BIT_DStream_t bitD;
FSE_DState_t state1;
FSE_DState_t state2;
{
nuint _var_err__ = BIT_initDStream(&bitD, cSrc, cSrcSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
for (; ((BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_unfinished) && (op < olimit)); op += 4)
{
op[0] = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if ((uint)((14 - 2) * 2 + 7) > (nuint)(sizeof(nuint)) * 8)
{
BIT_reloadDStream(&bitD);
}
op[1] = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
if ((uint)((14 - 2) * 4 + 7) > (nuint)(sizeof(nuint)) * 8)
{
if (BIT_reloadDStream(&bitD) > BIT_DStream_status.BIT_DStream_unfinished)
{
op += 2;
break;
}
}
op[2] = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if ((uint)((14 - 2) * 2 + 7) > (nuint)(sizeof(nuint)) * 8)
{
BIT_reloadDStream(&bitD);
}
op[3] = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
}
while (1 != 0)
{
if (op > (omax - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if (BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_overflow)
{
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
break;
}
if (op > (omax - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
if (BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_overflow)
{
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
break;
}
}
return (nuint)(op - ostart);
}
/*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_decompress_usingDTable(void* dst, nuint originalSize, void* cSrc, nuint cSrcSize, uint* dt)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
uint fastMode = DTableH->fastMode;
if (fastMode != 0)
{
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
}
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
public static nuint FSE_decompress_wksp(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_decompress_wksp_body(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize, int bmi2)
{
byte* istart = (byte*)(cSrc);
byte* ip = istart;
uint tableLog;
uint maxSymbolValue = 255;
FSE_DecompressWksp* wksp = (FSE_DecompressWksp*)(workSpace);
if (wkspSize < (nuint)(sizeof(FSE_DecompressWksp)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
{
nuint NCountLength = FSE_readNCount_bmi2((short*)wksp->ncount, &maxSymbolValue, &tableLog, (void*)istart, cSrcSize, bmi2);
if ((ERR_isError(NCountLength)) != 0)
{
return NCountLength;
}
if (tableLog > maxLog)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
assert(NCountLength <= cSrcSize);
ip += NCountLength;
cSrcSize -= NCountLength;
}
if ((((uint)((1 + (1 << (int)(tableLog)))) + ((((nuint)(sizeof(short)) * (maxSymbolValue + 1) + (1UL << (int)tableLog) + 8) + (nuint)(sizeof(uint)) - 1) / (nuint)(sizeof(uint))) + (uint)((255 + 1) / 2) + 1) * (nuint)(sizeof(uint))) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
workSpace = wksp->dtable + (1 + (1 << (int)(tableLog)));
wkspSize -= (nuint)(sizeof(FSE_DecompressWksp)) + ((uint)((1 + (1 << (int)(tableLog)))) * (nuint)(sizeof(uint)));
{
nuint _var_err__ = FSE_buildDTable_internal((uint*)wksp->dtable, (short*)wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
void* ptr = (void*)wksp->dtable;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
uint fastMode = DTableH->fastMode;
if (fastMode != 0)
{
return FSE_decompress_usingDTable_generic(dst, dstCapacity, (void*)ip, cSrcSize, (uint*)wksp->dtable, 1);
}
return FSE_decompress_usingDTable_generic(dst, dstCapacity, (void*)ip, cSrcSize, (uint*)wksp->dtable, 0);
}
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint FSE_decompress_wksp_body_default(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
}
private static nuint FSE_decompress_wksp_body_bmi2(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
}
public static nuint FSE_decompress_wksp_bmi2(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize, int bmi2)
{
if (bmi2 != 0)
{
return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
}
return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
}
/*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_buildDTable(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
uint* wksp = stackalloc uint[8322];
return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, (void*)wksp, (nuint)(sizeof(uint) * 8322));
}
/*! FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
into already allocated destination buffer 'dst', of size 'dstCapacity'.
@return : size of regenerated data (<= maxDstSize),
or an error code, which can be tested using FSE_isError() .
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases.
*/
public static nuint FSE_decompress(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize)
{
uint* wksp = stackalloc uint[5380];
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, (uint)((14 - 2)), (void*)wksp, (nuint)(sizeof(uint) * 5380));
}
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum HIST_checkInput_e
{
trustInput,
checkMaxSymbolValue,
}
}

View File

@@ -0,0 +1,13 @@
using System;
namespace ZstdSharp.Unsafe
{
/* static allocation of HUF's Compression Table */
/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
public partial struct HUF_CElt_s
{
public ushort val;
public byte nbBits;
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_CompressWeightsWksp
{
public fixed uint CTable[59];
public fixed uint scratchBuffer[30];
public fixed uint count[13];
public fixed short norm[13];
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-***************************/
/* single-symbol decoding */
/*-***************************/
public partial struct HUF_DEltX1
{
/* single-symbol decoding */
public byte @byte;
/* single-symbol decoding */
public byte nbBits;
}
}

View File

@@ -0,0 +1,19 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *************************/
/* double-symbols decoding */
/* *************************/
public partial struct HUF_DEltX2
{
/* double-symbols decoding */
public ushort sequence;
/* double-symbols decoding */
public byte nbBits;
/* double-symbols decoding */
public byte length;
}
}

View File

@@ -0,0 +1,17 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_ReadDTableX1_Workspace
{
public fixed uint rankVal[16];
public fixed uint rankStart[16];
public fixed uint statsWksp[218];
public fixed byte symbols[256];
public fixed byte huffWeight[256];
}
}

View File

@@ -0,0 +1,389 @@
using InlineIL;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static InlineIL.IL.Emit;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_ReadDTableX2_Workspace
{
public _rankVal_e__FixedBuffer rankVal;
public fixed uint rankStats[13];
public fixed uint rankStart0[14];
public _sortedSymbol_e__FixedBuffer sortedSymbol;
public fixed byte weightList[256];
public fixed uint calleeWksp[218];
public unsafe partial struct _rankVal_e__FixedBuffer
{
public rankValCol_t e0;
public rankValCol_t e1;
public rankValCol_t e2;
public rankValCol_t e3;
public rankValCol_t e4;
public rankValCol_t e5;
public rankValCol_t e6;
public rankValCol_t e7;
public rankValCol_t e8;
public rankValCol_t e9;
public rankValCol_t e10;
public rankValCol_t e11;
public ref rankValCol_t this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref rankValCol_t this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref rankValCol_t this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator rankValCol_t*(in _rankVal_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankVal_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<rankValCol_t>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static rankValCol_t* operator +(in _rankVal_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankVal_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<rankValCol_t>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<rankValCol_t>();
}
}
public unsafe partial struct _sortedSymbol_e__FixedBuffer
{
public sortedSymbol_t e0;
public sortedSymbol_t e1;
public sortedSymbol_t e2;
public sortedSymbol_t e3;
public sortedSymbol_t e4;
public sortedSymbol_t e5;
public sortedSymbol_t e6;
public sortedSymbol_t e7;
public sortedSymbol_t e8;
public sortedSymbol_t e9;
public sortedSymbol_t e10;
public sortedSymbol_t e11;
public sortedSymbol_t e12;
public sortedSymbol_t e13;
public sortedSymbol_t e14;
public sortedSymbol_t e15;
public sortedSymbol_t e16;
public sortedSymbol_t e17;
public sortedSymbol_t e18;
public sortedSymbol_t e19;
public sortedSymbol_t e20;
public sortedSymbol_t e21;
public sortedSymbol_t e22;
public sortedSymbol_t e23;
public sortedSymbol_t e24;
public sortedSymbol_t e25;
public sortedSymbol_t e26;
public sortedSymbol_t e27;
public sortedSymbol_t e28;
public sortedSymbol_t e29;
public sortedSymbol_t e30;
public sortedSymbol_t e31;
public sortedSymbol_t e32;
public sortedSymbol_t e33;
public sortedSymbol_t e34;
public sortedSymbol_t e35;
public sortedSymbol_t e36;
public sortedSymbol_t e37;
public sortedSymbol_t e38;
public sortedSymbol_t e39;
public sortedSymbol_t e40;
public sortedSymbol_t e41;
public sortedSymbol_t e42;
public sortedSymbol_t e43;
public sortedSymbol_t e44;
public sortedSymbol_t e45;
public sortedSymbol_t e46;
public sortedSymbol_t e47;
public sortedSymbol_t e48;
public sortedSymbol_t e49;
public sortedSymbol_t e50;
public sortedSymbol_t e51;
public sortedSymbol_t e52;
public sortedSymbol_t e53;
public sortedSymbol_t e54;
public sortedSymbol_t e55;
public sortedSymbol_t e56;
public sortedSymbol_t e57;
public sortedSymbol_t e58;
public sortedSymbol_t e59;
public sortedSymbol_t e60;
public sortedSymbol_t e61;
public sortedSymbol_t e62;
public sortedSymbol_t e63;
public sortedSymbol_t e64;
public sortedSymbol_t e65;
public sortedSymbol_t e66;
public sortedSymbol_t e67;
public sortedSymbol_t e68;
public sortedSymbol_t e69;
public sortedSymbol_t e70;
public sortedSymbol_t e71;
public sortedSymbol_t e72;
public sortedSymbol_t e73;
public sortedSymbol_t e74;
public sortedSymbol_t e75;
public sortedSymbol_t e76;
public sortedSymbol_t e77;
public sortedSymbol_t e78;
public sortedSymbol_t e79;
public sortedSymbol_t e80;
public sortedSymbol_t e81;
public sortedSymbol_t e82;
public sortedSymbol_t e83;
public sortedSymbol_t e84;
public sortedSymbol_t e85;
public sortedSymbol_t e86;
public sortedSymbol_t e87;
public sortedSymbol_t e88;
public sortedSymbol_t e89;
public sortedSymbol_t e90;
public sortedSymbol_t e91;
public sortedSymbol_t e92;
public sortedSymbol_t e93;
public sortedSymbol_t e94;
public sortedSymbol_t e95;
public sortedSymbol_t e96;
public sortedSymbol_t e97;
public sortedSymbol_t e98;
public sortedSymbol_t e99;
public sortedSymbol_t e100;
public sortedSymbol_t e101;
public sortedSymbol_t e102;
public sortedSymbol_t e103;
public sortedSymbol_t e104;
public sortedSymbol_t e105;
public sortedSymbol_t e106;
public sortedSymbol_t e107;
public sortedSymbol_t e108;
public sortedSymbol_t e109;
public sortedSymbol_t e110;
public sortedSymbol_t e111;
public sortedSymbol_t e112;
public sortedSymbol_t e113;
public sortedSymbol_t e114;
public sortedSymbol_t e115;
public sortedSymbol_t e116;
public sortedSymbol_t e117;
public sortedSymbol_t e118;
public sortedSymbol_t e119;
public sortedSymbol_t e120;
public sortedSymbol_t e121;
public sortedSymbol_t e122;
public sortedSymbol_t e123;
public sortedSymbol_t e124;
public sortedSymbol_t e125;
public sortedSymbol_t e126;
public sortedSymbol_t e127;
public sortedSymbol_t e128;
public sortedSymbol_t e129;
public sortedSymbol_t e130;
public sortedSymbol_t e131;
public sortedSymbol_t e132;
public sortedSymbol_t e133;
public sortedSymbol_t e134;
public sortedSymbol_t e135;
public sortedSymbol_t e136;
public sortedSymbol_t e137;
public sortedSymbol_t e138;
public sortedSymbol_t e139;
public sortedSymbol_t e140;
public sortedSymbol_t e141;
public sortedSymbol_t e142;
public sortedSymbol_t e143;
public sortedSymbol_t e144;
public sortedSymbol_t e145;
public sortedSymbol_t e146;
public sortedSymbol_t e147;
public sortedSymbol_t e148;
public sortedSymbol_t e149;
public sortedSymbol_t e150;
public sortedSymbol_t e151;
public sortedSymbol_t e152;
public sortedSymbol_t e153;
public sortedSymbol_t e154;
public sortedSymbol_t e155;
public sortedSymbol_t e156;
public sortedSymbol_t e157;
public sortedSymbol_t e158;
public sortedSymbol_t e159;
public sortedSymbol_t e160;
public sortedSymbol_t e161;
public sortedSymbol_t e162;
public sortedSymbol_t e163;
public sortedSymbol_t e164;
public sortedSymbol_t e165;
public sortedSymbol_t e166;
public sortedSymbol_t e167;
public sortedSymbol_t e168;
public sortedSymbol_t e169;
public sortedSymbol_t e170;
public sortedSymbol_t e171;
public sortedSymbol_t e172;
public sortedSymbol_t e173;
public sortedSymbol_t e174;
public sortedSymbol_t e175;
public sortedSymbol_t e176;
public sortedSymbol_t e177;
public sortedSymbol_t e178;
public sortedSymbol_t e179;
public sortedSymbol_t e180;
public sortedSymbol_t e181;
public sortedSymbol_t e182;
public sortedSymbol_t e183;
public sortedSymbol_t e184;
public sortedSymbol_t e185;
public sortedSymbol_t e186;
public sortedSymbol_t e187;
public sortedSymbol_t e188;
public sortedSymbol_t e189;
public sortedSymbol_t e190;
public sortedSymbol_t e191;
public sortedSymbol_t e192;
public sortedSymbol_t e193;
public sortedSymbol_t e194;
public sortedSymbol_t e195;
public sortedSymbol_t e196;
public sortedSymbol_t e197;
public sortedSymbol_t e198;
public sortedSymbol_t e199;
public sortedSymbol_t e200;
public sortedSymbol_t e201;
public sortedSymbol_t e202;
public sortedSymbol_t e203;
public sortedSymbol_t e204;
public sortedSymbol_t e205;
public sortedSymbol_t e206;
public sortedSymbol_t e207;
public sortedSymbol_t e208;
public sortedSymbol_t e209;
public sortedSymbol_t e210;
public sortedSymbol_t e211;
public sortedSymbol_t e212;
public sortedSymbol_t e213;
public sortedSymbol_t e214;
public sortedSymbol_t e215;
public sortedSymbol_t e216;
public sortedSymbol_t e217;
public sortedSymbol_t e218;
public sortedSymbol_t e219;
public sortedSymbol_t e220;
public sortedSymbol_t e221;
public sortedSymbol_t e222;
public sortedSymbol_t e223;
public sortedSymbol_t e224;
public sortedSymbol_t e225;
public sortedSymbol_t e226;
public sortedSymbol_t e227;
public sortedSymbol_t e228;
public sortedSymbol_t e229;
public sortedSymbol_t e230;
public sortedSymbol_t e231;
public sortedSymbol_t e232;
public sortedSymbol_t e233;
public sortedSymbol_t e234;
public sortedSymbol_t e235;
public sortedSymbol_t e236;
public sortedSymbol_t e237;
public sortedSymbol_t e238;
public sortedSymbol_t e239;
public sortedSymbol_t e240;
public sortedSymbol_t e241;
public sortedSymbol_t e242;
public sortedSymbol_t e243;
public sortedSymbol_t e244;
public sortedSymbol_t e245;
public sortedSymbol_t e246;
public sortedSymbol_t e247;
public sortedSymbol_t e248;
public sortedSymbol_t e249;
public sortedSymbol_t e250;
public sortedSymbol_t e251;
public sortedSymbol_t e252;
public sortedSymbol_t e253;
public sortedSymbol_t e254;
public sortedSymbol_t e255;
public ref sortedSymbol_t this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref sortedSymbol_t this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref sortedSymbol_t this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator sortedSymbol_t*(in _sortedSymbol_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_sortedSymbol_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<sortedSymbol_t>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static sortedSymbol_t* operator +(in _sortedSymbol_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_sortedSymbol_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<sortedSymbol_t>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<sortedSymbol_t>();
}
}
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_WriteCTableWksp
{
public HUF_CompressWeightsWksp wksp;
/* precomputed conversion table */
public fixed byte bitsToWeight[13];
public fixed byte huffWeight[255];
}
}

View File

@@ -0,0 +1,657 @@
using InlineIL;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static InlineIL.IL.Emit;
namespace ZstdSharp.Unsafe
{
public partial struct HUF_buildCTable_wksp_tables
{
public _huffNodeTbl_e__FixedBuffer huffNodeTbl;
public _rankPosition_e__FixedBuffer rankPosition;
public unsafe partial struct _huffNodeTbl_e__FixedBuffer
{
public nodeElt_s e0;
public nodeElt_s e1;
public nodeElt_s e2;
public nodeElt_s e3;
public nodeElt_s e4;
public nodeElt_s e5;
public nodeElt_s e6;
public nodeElt_s e7;
public nodeElt_s e8;
public nodeElt_s e9;
public nodeElt_s e10;
public nodeElt_s e11;
public nodeElt_s e12;
public nodeElt_s e13;
public nodeElt_s e14;
public nodeElt_s e15;
public nodeElt_s e16;
public nodeElt_s e17;
public nodeElt_s e18;
public nodeElt_s e19;
public nodeElt_s e20;
public nodeElt_s e21;
public nodeElt_s e22;
public nodeElt_s e23;
public nodeElt_s e24;
public nodeElt_s e25;
public nodeElt_s e26;
public nodeElt_s e27;
public nodeElt_s e28;
public nodeElt_s e29;
public nodeElt_s e30;
public nodeElt_s e31;
public nodeElt_s e32;
public nodeElt_s e33;
public nodeElt_s e34;
public nodeElt_s e35;
public nodeElt_s e36;
public nodeElt_s e37;
public nodeElt_s e38;
public nodeElt_s e39;
public nodeElt_s e40;
public nodeElt_s e41;
public nodeElt_s e42;
public nodeElt_s e43;
public nodeElt_s e44;
public nodeElt_s e45;
public nodeElt_s e46;
public nodeElt_s e47;
public nodeElt_s e48;
public nodeElt_s e49;
public nodeElt_s e50;
public nodeElt_s e51;
public nodeElt_s e52;
public nodeElt_s e53;
public nodeElt_s e54;
public nodeElt_s e55;
public nodeElt_s e56;
public nodeElt_s e57;
public nodeElt_s e58;
public nodeElt_s e59;
public nodeElt_s e60;
public nodeElt_s e61;
public nodeElt_s e62;
public nodeElt_s e63;
public nodeElt_s e64;
public nodeElt_s e65;
public nodeElt_s e66;
public nodeElt_s e67;
public nodeElt_s e68;
public nodeElt_s e69;
public nodeElt_s e70;
public nodeElt_s e71;
public nodeElt_s e72;
public nodeElt_s e73;
public nodeElt_s e74;
public nodeElt_s e75;
public nodeElt_s e76;
public nodeElt_s e77;
public nodeElt_s e78;
public nodeElt_s e79;
public nodeElt_s e80;
public nodeElt_s e81;
public nodeElt_s e82;
public nodeElt_s e83;
public nodeElt_s e84;
public nodeElt_s e85;
public nodeElt_s e86;
public nodeElt_s e87;
public nodeElt_s e88;
public nodeElt_s e89;
public nodeElt_s e90;
public nodeElt_s e91;
public nodeElt_s e92;
public nodeElt_s e93;
public nodeElt_s e94;
public nodeElt_s e95;
public nodeElt_s e96;
public nodeElt_s e97;
public nodeElt_s e98;
public nodeElt_s e99;
public nodeElt_s e100;
public nodeElt_s e101;
public nodeElt_s e102;
public nodeElt_s e103;
public nodeElt_s e104;
public nodeElt_s e105;
public nodeElt_s e106;
public nodeElt_s e107;
public nodeElt_s e108;
public nodeElt_s e109;
public nodeElt_s e110;
public nodeElt_s e111;
public nodeElt_s e112;
public nodeElt_s e113;
public nodeElt_s e114;
public nodeElt_s e115;
public nodeElt_s e116;
public nodeElt_s e117;
public nodeElt_s e118;
public nodeElt_s e119;
public nodeElt_s e120;
public nodeElt_s e121;
public nodeElt_s e122;
public nodeElt_s e123;
public nodeElt_s e124;
public nodeElt_s e125;
public nodeElt_s e126;
public nodeElt_s e127;
public nodeElt_s e128;
public nodeElt_s e129;
public nodeElt_s e130;
public nodeElt_s e131;
public nodeElt_s e132;
public nodeElt_s e133;
public nodeElt_s e134;
public nodeElt_s e135;
public nodeElt_s e136;
public nodeElt_s e137;
public nodeElt_s e138;
public nodeElt_s e139;
public nodeElt_s e140;
public nodeElt_s e141;
public nodeElt_s e142;
public nodeElt_s e143;
public nodeElt_s e144;
public nodeElt_s e145;
public nodeElt_s e146;
public nodeElt_s e147;
public nodeElt_s e148;
public nodeElt_s e149;
public nodeElt_s e150;
public nodeElt_s e151;
public nodeElt_s e152;
public nodeElt_s e153;
public nodeElt_s e154;
public nodeElt_s e155;
public nodeElt_s e156;
public nodeElt_s e157;
public nodeElt_s e158;
public nodeElt_s e159;
public nodeElt_s e160;
public nodeElt_s e161;
public nodeElt_s e162;
public nodeElt_s e163;
public nodeElt_s e164;
public nodeElt_s e165;
public nodeElt_s e166;
public nodeElt_s e167;
public nodeElt_s e168;
public nodeElt_s e169;
public nodeElt_s e170;
public nodeElt_s e171;
public nodeElt_s e172;
public nodeElt_s e173;
public nodeElt_s e174;
public nodeElt_s e175;
public nodeElt_s e176;
public nodeElt_s e177;
public nodeElt_s e178;
public nodeElt_s e179;
public nodeElt_s e180;
public nodeElt_s e181;
public nodeElt_s e182;
public nodeElt_s e183;
public nodeElt_s e184;
public nodeElt_s e185;
public nodeElt_s e186;
public nodeElt_s e187;
public nodeElt_s e188;
public nodeElt_s e189;
public nodeElt_s e190;
public nodeElt_s e191;
public nodeElt_s e192;
public nodeElt_s e193;
public nodeElt_s e194;
public nodeElt_s e195;
public nodeElt_s e196;
public nodeElt_s e197;
public nodeElt_s e198;
public nodeElt_s e199;
public nodeElt_s e200;
public nodeElt_s e201;
public nodeElt_s e202;
public nodeElt_s e203;
public nodeElt_s e204;
public nodeElt_s e205;
public nodeElt_s e206;
public nodeElt_s e207;
public nodeElt_s e208;
public nodeElt_s e209;
public nodeElt_s e210;
public nodeElt_s e211;
public nodeElt_s e212;
public nodeElt_s e213;
public nodeElt_s e214;
public nodeElt_s e215;
public nodeElt_s e216;
public nodeElt_s e217;
public nodeElt_s e218;
public nodeElt_s e219;
public nodeElt_s e220;
public nodeElt_s e221;
public nodeElt_s e222;
public nodeElt_s e223;
public nodeElt_s e224;
public nodeElt_s e225;
public nodeElt_s e226;
public nodeElt_s e227;
public nodeElt_s e228;
public nodeElt_s e229;
public nodeElt_s e230;
public nodeElt_s e231;
public nodeElt_s e232;
public nodeElt_s e233;
public nodeElt_s e234;
public nodeElt_s e235;
public nodeElt_s e236;
public nodeElt_s e237;
public nodeElt_s e238;
public nodeElt_s e239;
public nodeElt_s e240;
public nodeElt_s e241;
public nodeElt_s e242;
public nodeElt_s e243;
public nodeElt_s e244;
public nodeElt_s e245;
public nodeElt_s e246;
public nodeElt_s e247;
public nodeElt_s e248;
public nodeElt_s e249;
public nodeElt_s e250;
public nodeElt_s e251;
public nodeElt_s e252;
public nodeElt_s e253;
public nodeElt_s e254;
public nodeElt_s e255;
public nodeElt_s e256;
public nodeElt_s e257;
public nodeElt_s e258;
public nodeElt_s e259;
public nodeElt_s e260;
public nodeElt_s e261;
public nodeElt_s e262;
public nodeElt_s e263;
public nodeElt_s e264;
public nodeElt_s e265;
public nodeElt_s e266;
public nodeElt_s e267;
public nodeElt_s e268;
public nodeElt_s e269;
public nodeElt_s e270;
public nodeElt_s e271;
public nodeElt_s e272;
public nodeElt_s e273;
public nodeElt_s e274;
public nodeElt_s e275;
public nodeElt_s e276;
public nodeElt_s e277;
public nodeElt_s e278;
public nodeElt_s e279;
public nodeElt_s e280;
public nodeElt_s e281;
public nodeElt_s e282;
public nodeElt_s e283;
public nodeElt_s e284;
public nodeElt_s e285;
public nodeElt_s e286;
public nodeElt_s e287;
public nodeElt_s e288;
public nodeElt_s e289;
public nodeElt_s e290;
public nodeElt_s e291;
public nodeElt_s e292;
public nodeElt_s e293;
public nodeElt_s e294;
public nodeElt_s e295;
public nodeElt_s e296;
public nodeElt_s e297;
public nodeElt_s e298;
public nodeElt_s e299;
public nodeElt_s e300;
public nodeElt_s e301;
public nodeElt_s e302;
public nodeElt_s e303;
public nodeElt_s e304;
public nodeElt_s e305;
public nodeElt_s e306;
public nodeElt_s e307;
public nodeElt_s e308;
public nodeElt_s e309;
public nodeElt_s e310;
public nodeElt_s e311;
public nodeElt_s e312;
public nodeElt_s e313;
public nodeElt_s e314;
public nodeElt_s e315;
public nodeElt_s e316;
public nodeElt_s e317;
public nodeElt_s e318;
public nodeElt_s e319;
public nodeElt_s e320;
public nodeElt_s e321;
public nodeElt_s e322;
public nodeElt_s e323;
public nodeElt_s e324;
public nodeElt_s e325;
public nodeElt_s e326;
public nodeElt_s e327;
public nodeElt_s e328;
public nodeElt_s e329;
public nodeElt_s e330;
public nodeElt_s e331;
public nodeElt_s e332;
public nodeElt_s e333;
public nodeElt_s e334;
public nodeElt_s e335;
public nodeElt_s e336;
public nodeElt_s e337;
public nodeElt_s e338;
public nodeElt_s e339;
public nodeElt_s e340;
public nodeElt_s e341;
public nodeElt_s e342;
public nodeElt_s e343;
public nodeElt_s e344;
public nodeElt_s e345;
public nodeElt_s e346;
public nodeElt_s e347;
public nodeElt_s e348;
public nodeElt_s e349;
public nodeElt_s e350;
public nodeElt_s e351;
public nodeElt_s e352;
public nodeElt_s e353;
public nodeElt_s e354;
public nodeElt_s e355;
public nodeElt_s e356;
public nodeElt_s e357;
public nodeElt_s e358;
public nodeElt_s e359;
public nodeElt_s e360;
public nodeElt_s e361;
public nodeElt_s e362;
public nodeElt_s e363;
public nodeElt_s e364;
public nodeElt_s e365;
public nodeElt_s e366;
public nodeElt_s e367;
public nodeElt_s e368;
public nodeElt_s e369;
public nodeElt_s e370;
public nodeElt_s e371;
public nodeElt_s e372;
public nodeElt_s e373;
public nodeElt_s e374;
public nodeElt_s e375;
public nodeElt_s e376;
public nodeElt_s e377;
public nodeElt_s e378;
public nodeElt_s e379;
public nodeElt_s e380;
public nodeElt_s e381;
public nodeElt_s e382;
public nodeElt_s e383;
public nodeElt_s e384;
public nodeElt_s e385;
public nodeElt_s e386;
public nodeElt_s e387;
public nodeElt_s e388;
public nodeElt_s e389;
public nodeElt_s e390;
public nodeElt_s e391;
public nodeElt_s e392;
public nodeElt_s e393;
public nodeElt_s e394;
public nodeElt_s e395;
public nodeElt_s e396;
public nodeElt_s e397;
public nodeElt_s e398;
public nodeElt_s e399;
public nodeElt_s e400;
public nodeElt_s e401;
public nodeElt_s e402;
public nodeElt_s e403;
public nodeElt_s e404;
public nodeElt_s e405;
public nodeElt_s e406;
public nodeElt_s e407;
public nodeElt_s e408;
public nodeElt_s e409;
public nodeElt_s e410;
public nodeElt_s e411;
public nodeElt_s e412;
public nodeElt_s e413;
public nodeElt_s e414;
public nodeElt_s e415;
public nodeElt_s e416;
public nodeElt_s e417;
public nodeElt_s e418;
public nodeElt_s e419;
public nodeElt_s e420;
public nodeElt_s e421;
public nodeElt_s e422;
public nodeElt_s e423;
public nodeElt_s e424;
public nodeElt_s e425;
public nodeElt_s e426;
public nodeElt_s e427;
public nodeElt_s e428;
public nodeElt_s e429;
public nodeElt_s e430;
public nodeElt_s e431;
public nodeElt_s e432;
public nodeElt_s e433;
public nodeElt_s e434;
public nodeElt_s e435;
public nodeElt_s e436;
public nodeElt_s e437;
public nodeElt_s e438;
public nodeElt_s e439;
public nodeElt_s e440;
public nodeElt_s e441;
public nodeElt_s e442;
public nodeElt_s e443;
public nodeElt_s e444;
public nodeElt_s e445;
public nodeElt_s e446;
public nodeElt_s e447;
public nodeElt_s e448;
public nodeElt_s e449;
public nodeElt_s e450;
public nodeElt_s e451;
public nodeElt_s e452;
public nodeElt_s e453;
public nodeElt_s e454;
public nodeElt_s e455;
public nodeElt_s e456;
public nodeElt_s e457;
public nodeElt_s e458;
public nodeElt_s e459;
public nodeElt_s e460;
public nodeElt_s e461;
public nodeElt_s e462;
public nodeElt_s e463;
public nodeElt_s e464;
public nodeElt_s e465;
public nodeElt_s e466;
public nodeElt_s e467;
public nodeElt_s e468;
public nodeElt_s e469;
public nodeElt_s e470;
public nodeElt_s e471;
public nodeElt_s e472;
public nodeElt_s e473;
public nodeElt_s e474;
public nodeElt_s e475;
public nodeElt_s e476;
public nodeElt_s e477;
public nodeElt_s e478;
public nodeElt_s e479;
public nodeElt_s e480;
public nodeElt_s e481;
public nodeElt_s e482;
public nodeElt_s e483;
public nodeElt_s e484;
public nodeElt_s e485;
public nodeElt_s e486;
public nodeElt_s e487;
public nodeElt_s e488;
public nodeElt_s e489;
public nodeElt_s e490;
public nodeElt_s e491;
public nodeElt_s e492;
public nodeElt_s e493;
public nodeElt_s e494;
public nodeElt_s e495;
public nodeElt_s e496;
public nodeElt_s e497;
public nodeElt_s e498;
public nodeElt_s e499;
public nodeElt_s e500;
public nodeElt_s e501;
public nodeElt_s e502;
public nodeElt_s e503;
public nodeElt_s e504;
public nodeElt_s e505;
public nodeElt_s e506;
public nodeElt_s e507;
public nodeElt_s e508;
public nodeElt_s e509;
public nodeElt_s e510;
public nodeElt_s e511;
public ref nodeElt_s this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref nodeElt_s this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref nodeElt_s this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator nodeElt_s*(in _huffNodeTbl_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_huffNodeTbl_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<nodeElt_s>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static nodeElt_s* operator +(in _huffNodeTbl_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_huffNodeTbl_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<nodeElt_s>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<nodeElt_s>();
}
}
public unsafe partial struct _rankPosition_e__FixedBuffer
{
public rankPos e0;
public rankPos e1;
public rankPos e2;
public rankPos e3;
public rankPos e4;
public rankPos e5;
public rankPos e6;
public rankPos e7;
public rankPos e8;
public rankPos e9;
public rankPos e10;
public rankPos e11;
public rankPos e12;
public rankPos e13;
public rankPos e14;
public rankPos e15;
public rankPos e16;
public rankPos e17;
public rankPos e18;
public rankPos e19;
public rankPos e20;
public rankPos e21;
public rankPos e22;
public rankPos e23;
public rankPos e24;
public rankPos e25;
public rankPos e26;
public rankPos e27;
public rankPos e28;
public rankPos e29;
public rankPos e30;
public rankPos e31;
public ref rankPos this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref rankPos this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref rankPos this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator rankPos*(in _rankPosition_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankPosition_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<rankPos>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static rankPos* operator +(in _rankPosition_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankPosition_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<rankPos>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<rankPos>();
}
}
}
}

View File

@@ -0,0 +1,322 @@
using InlineIL;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static InlineIL.IL.Emit;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_compress_tables_t
{
public fixed uint count[256];
public _CTable_e__FixedBuffer CTable;
public _wksps_e__Union wksps;
public unsafe partial struct _CTable_e__FixedBuffer
{
public HUF_CElt_s e0;
public HUF_CElt_s e1;
public HUF_CElt_s e2;
public HUF_CElt_s e3;
public HUF_CElt_s e4;
public HUF_CElt_s e5;
public HUF_CElt_s e6;
public HUF_CElt_s e7;
public HUF_CElt_s e8;
public HUF_CElt_s e9;
public HUF_CElt_s e10;
public HUF_CElt_s e11;
public HUF_CElt_s e12;
public HUF_CElt_s e13;
public HUF_CElt_s e14;
public HUF_CElt_s e15;
public HUF_CElt_s e16;
public HUF_CElt_s e17;
public HUF_CElt_s e18;
public HUF_CElt_s e19;
public HUF_CElt_s e20;
public HUF_CElt_s e21;
public HUF_CElt_s e22;
public HUF_CElt_s e23;
public HUF_CElt_s e24;
public HUF_CElt_s e25;
public HUF_CElt_s e26;
public HUF_CElt_s e27;
public HUF_CElt_s e28;
public HUF_CElt_s e29;
public HUF_CElt_s e30;
public HUF_CElt_s e31;
public HUF_CElt_s e32;
public HUF_CElt_s e33;
public HUF_CElt_s e34;
public HUF_CElt_s e35;
public HUF_CElt_s e36;
public HUF_CElt_s e37;
public HUF_CElt_s e38;
public HUF_CElt_s e39;
public HUF_CElt_s e40;
public HUF_CElt_s e41;
public HUF_CElt_s e42;
public HUF_CElt_s e43;
public HUF_CElt_s e44;
public HUF_CElt_s e45;
public HUF_CElt_s e46;
public HUF_CElt_s e47;
public HUF_CElt_s e48;
public HUF_CElt_s e49;
public HUF_CElt_s e50;
public HUF_CElt_s e51;
public HUF_CElt_s e52;
public HUF_CElt_s e53;
public HUF_CElt_s e54;
public HUF_CElt_s e55;
public HUF_CElt_s e56;
public HUF_CElt_s e57;
public HUF_CElt_s e58;
public HUF_CElt_s e59;
public HUF_CElt_s e60;
public HUF_CElt_s e61;
public HUF_CElt_s e62;
public HUF_CElt_s e63;
public HUF_CElt_s e64;
public HUF_CElt_s e65;
public HUF_CElt_s e66;
public HUF_CElt_s e67;
public HUF_CElt_s e68;
public HUF_CElt_s e69;
public HUF_CElt_s e70;
public HUF_CElt_s e71;
public HUF_CElt_s e72;
public HUF_CElt_s e73;
public HUF_CElt_s e74;
public HUF_CElt_s e75;
public HUF_CElt_s e76;
public HUF_CElt_s e77;
public HUF_CElt_s e78;
public HUF_CElt_s e79;
public HUF_CElt_s e80;
public HUF_CElt_s e81;
public HUF_CElt_s e82;
public HUF_CElt_s e83;
public HUF_CElt_s e84;
public HUF_CElt_s e85;
public HUF_CElt_s e86;
public HUF_CElt_s e87;
public HUF_CElt_s e88;
public HUF_CElt_s e89;
public HUF_CElt_s e90;
public HUF_CElt_s e91;
public HUF_CElt_s e92;
public HUF_CElt_s e93;
public HUF_CElt_s e94;
public HUF_CElt_s e95;
public HUF_CElt_s e96;
public HUF_CElt_s e97;
public HUF_CElt_s e98;
public HUF_CElt_s e99;
public HUF_CElt_s e100;
public HUF_CElt_s e101;
public HUF_CElt_s e102;
public HUF_CElt_s e103;
public HUF_CElt_s e104;
public HUF_CElt_s e105;
public HUF_CElt_s e106;
public HUF_CElt_s e107;
public HUF_CElt_s e108;
public HUF_CElt_s e109;
public HUF_CElt_s e110;
public HUF_CElt_s e111;
public HUF_CElt_s e112;
public HUF_CElt_s e113;
public HUF_CElt_s e114;
public HUF_CElt_s e115;
public HUF_CElt_s e116;
public HUF_CElt_s e117;
public HUF_CElt_s e118;
public HUF_CElt_s e119;
public HUF_CElt_s e120;
public HUF_CElt_s e121;
public HUF_CElt_s e122;
public HUF_CElt_s e123;
public HUF_CElt_s e124;
public HUF_CElt_s e125;
public HUF_CElt_s e126;
public HUF_CElt_s e127;
public HUF_CElt_s e128;
public HUF_CElt_s e129;
public HUF_CElt_s e130;
public HUF_CElt_s e131;
public HUF_CElt_s e132;
public HUF_CElt_s e133;
public HUF_CElt_s e134;
public HUF_CElt_s e135;
public HUF_CElt_s e136;
public HUF_CElt_s e137;
public HUF_CElt_s e138;
public HUF_CElt_s e139;
public HUF_CElt_s e140;
public HUF_CElt_s e141;
public HUF_CElt_s e142;
public HUF_CElt_s e143;
public HUF_CElt_s e144;
public HUF_CElt_s e145;
public HUF_CElt_s e146;
public HUF_CElt_s e147;
public HUF_CElt_s e148;
public HUF_CElt_s e149;
public HUF_CElt_s e150;
public HUF_CElt_s e151;
public HUF_CElt_s e152;
public HUF_CElt_s e153;
public HUF_CElt_s e154;
public HUF_CElt_s e155;
public HUF_CElt_s e156;
public HUF_CElt_s e157;
public HUF_CElt_s e158;
public HUF_CElt_s e159;
public HUF_CElt_s e160;
public HUF_CElt_s e161;
public HUF_CElt_s e162;
public HUF_CElt_s e163;
public HUF_CElt_s e164;
public HUF_CElt_s e165;
public HUF_CElt_s e166;
public HUF_CElt_s e167;
public HUF_CElt_s e168;
public HUF_CElt_s e169;
public HUF_CElt_s e170;
public HUF_CElt_s e171;
public HUF_CElt_s e172;
public HUF_CElt_s e173;
public HUF_CElt_s e174;
public HUF_CElt_s e175;
public HUF_CElt_s e176;
public HUF_CElt_s e177;
public HUF_CElt_s e178;
public HUF_CElt_s e179;
public HUF_CElt_s e180;
public HUF_CElt_s e181;
public HUF_CElt_s e182;
public HUF_CElt_s e183;
public HUF_CElt_s e184;
public HUF_CElt_s e185;
public HUF_CElt_s e186;
public HUF_CElt_s e187;
public HUF_CElt_s e188;
public HUF_CElt_s e189;
public HUF_CElt_s e190;
public HUF_CElt_s e191;
public HUF_CElt_s e192;
public HUF_CElt_s e193;
public HUF_CElt_s e194;
public HUF_CElt_s e195;
public HUF_CElt_s e196;
public HUF_CElt_s e197;
public HUF_CElt_s e198;
public HUF_CElt_s e199;
public HUF_CElt_s e200;
public HUF_CElt_s e201;
public HUF_CElt_s e202;
public HUF_CElt_s e203;
public HUF_CElt_s e204;
public HUF_CElt_s e205;
public HUF_CElt_s e206;
public HUF_CElt_s e207;
public HUF_CElt_s e208;
public HUF_CElt_s e209;
public HUF_CElt_s e210;
public HUF_CElt_s e211;
public HUF_CElt_s e212;
public HUF_CElt_s e213;
public HUF_CElt_s e214;
public HUF_CElt_s e215;
public HUF_CElt_s e216;
public HUF_CElt_s e217;
public HUF_CElt_s e218;
public HUF_CElt_s e219;
public HUF_CElt_s e220;
public HUF_CElt_s e221;
public HUF_CElt_s e222;
public HUF_CElt_s e223;
public HUF_CElt_s e224;
public HUF_CElt_s e225;
public HUF_CElt_s e226;
public HUF_CElt_s e227;
public HUF_CElt_s e228;
public HUF_CElt_s e229;
public HUF_CElt_s e230;
public HUF_CElt_s e231;
public HUF_CElt_s e232;
public HUF_CElt_s e233;
public HUF_CElt_s e234;
public HUF_CElt_s e235;
public HUF_CElt_s e236;
public HUF_CElt_s e237;
public HUF_CElt_s e238;
public HUF_CElt_s e239;
public HUF_CElt_s e240;
public HUF_CElt_s e241;
public HUF_CElt_s e242;
public HUF_CElt_s e243;
public HUF_CElt_s e244;
public HUF_CElt_s e245;
public HUF_CElt_s e246;
public HUF_CElt_s e247;
public HUF_CElt_s e248;
public HUF_CElt_s e249;
public HUF_CElt_s e250;
public HUF_CElt_s e251;
public HUF_CElt_s e252;
public HUF_CElt_s e253;
public HUF_CElt_s e254;
public HUF_CElt_s e255;
public ref HUF_CElt_s this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref HUF_CElt_s this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref HUF_CElt_s this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator HUF_CElt_s*(in _CTable_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_CTable_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<HUF_CElt_s>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static HUF_CElt_s* operator +(in _CTable_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_CTable_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<HUF_CElt_s>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<HUF_CElt_s>();
}
}
}
}

View File

@@ -0,0 +1,8 @@
using System;
using System.Runtime.InteropServices;
namespace ZstdSharp.Unsafe
{
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
public unsafe delegate nuint HUF_decompress_usingDTable_t(void* dst, nuint dstSize, void* cSrc, nuint cSrcSize, uint* DTable);
}

View File

@@ -0,0 +1,10 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum HUF_nbStreams_e
{
HUF_singleStream,
HUF_fourStreams,
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum HUF_repeat
{
HUF_repeat_none,
HUF_repeat_check,
HUF_repeat_valid,
}
}

View File

@@ -0,0 +1,240 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/* --- Error management --- */
public static uint HIST_isError(nuint code)
{
return ERR_isError(code);
}
/*-**************************************************************
* Histogram functions
****************************************************************/
public static uint HIST_count_simple(uint* count, uint* maxSymbolValuePtr, void* src, nuint srcSize)
{
byte* ip = (byte*)(src);
byte* end = ip + srcSize;
uint maxSymbolValue = *maxSymbolValuePtr;
uint largestCount = 0;
memset((void*)(count), (0), ((maxSymbolValue + 1) * (nuint)(sizeof(uint))));
if (srcSize == 0)
{
*maxSymbolValuePtr = 0;
return 0;
}
while (ip < end)
{
assert(*ip <= maxSymbolValue);
count[*ip++]++;
}
while ((count[maxSymbolValue]) == 0)
{
maxSymbolValue--;
}
*maxSymbolValuePtr = maxSymbolValue;
{
uint s;
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] > largestCount)
{
largestCount = count[s];
}
}
}
return largestCount;
}
/* HIST_count_parallel_wksp() :
* store histogram into 4 intermediate tables, recombined at the end.
* this design makes better use of OoO cpus,
* and is noticeably faster when some values are heavily repeated.
* But it needs some additional workspace for intermediate tables.
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
* @return : largest histogram frequency,
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
private static nuint HIST_count_parallel_wksp(uint* count, uint* maxSymbolValuePtr, void* source, nuint sourceSize, HIST_checkInput_e check, uint* workSpace)
{
byte* ip = (byte*)(source);
byte* iend = ip + sourceSize;
nuint countSize = (*maxSymbolValuePtr + 1) * (nuint)(4);
uint max = 0;
uint* Counting1 = workSpace;
uint* Counting2 = Counting1 + 256;
uint* Counting3 = Counting2 + 256;
uint* Counting4 = Counting3 + 256;
assert(*maxSymbolValuePtr <= 255);
if (sourceSize == 0)
{
memset((void*)(count), (0), (countSize));
*maxSymbolValuePtr = 0;
return 0;
}
memset((void*)(workSpace), (0), ((uint)(4 * 256) * (nuint)(sizeof(uint))));
{
uint cached = MEM_read32((void*)ip);
ip += 4;
while (ip < iend - 15)
{
uint c = cached;
cached = MEM_read32((void*)ip);
ip += 4;
Counting1[(byte)(c)]++;
Counting2[(byte)(c >> 8)]++;
Counting3[(byte)(c >> 16)]++;
Counting4[c >> 24]++;
c = cached;
cached = MEM_read32((void*)ip);
ip += 4;
Counting1[(byte)(c)]++;
Counting2[(byte)(c >> 8)]++;
Counting3[(byte)(c >> 16)]++;
Counting4[c >> 24]++;
c = cached;
cached = MEM_read32((void*)ip);
ip += 4;
Counting1[(byte)(c)]++;
Counting2[(byte)(c >> 8)]++;
Counting3[(byte)(c >> 16)]++;
Counting4[c >> 24]++;
c = cached;
cached = MEM_read32((void*)ip);
ip += 4;
Counting1[(byte)(c)]++;
Counting2[(byte)(c >> 8)]++;
Counting3[(byte)(c >> 16)]++;
Counting4[c >> 24]++;
}
ip -= 4;
}
while (ip < iend)
{
Counting1[*ip++]++;
}
{
uint s;
for (s = 0; s < 256; s++)
{
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s] > max)
{
max = Counting1[s];
}
}
}
{
uint maxSymbolValue = 255;
while ((Counting1[maxSymbolValue]) == 0)
{
maxSymbolValue--;
}
if (check != default && maxSymbolValue > *maxSymbolValuePtr)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall)));
}
*maxSymbolValuePtr = maxSymbolValue;
memmove((void*)(count), (void*)(Counting1), (countSize));
}
return (nuint)(max);
}
/* HIST_countFast_wksp() :
* Same as HIST_countFast(), but using an externally provided scratch buffer.
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
public static nuint HIST_countFast_wksp(uint* count, uint* maxSymbolValuePtr, void* source, nuint sourceSize, void* workSpace, nuint workSpaceSize)
{
if (sourceSize < 1500)
{
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
}
if (((nuint)(workSpace) & 3) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (workSpaceSize < (1024 * (nuint)(sizeof(uint))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall)));
}
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, HIST_checkInput_e.trustInput, (uint*)(workSpace));
}
/* HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
public static nuint HIST_count_wksp(uint* count, uint* maxSymbolValuePtr, void* source, nuint sourceSize, void* workSpace, nuint workSpaceSize)
{
if (((nuint)(workSpace) & 3) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (workSpaceSize < (1024 * (nuint)(sizeof(uint))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall)));
}
if (*maxSymbolValuePtr < 255)
{
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, HIST_checkInput_e.checkMaxSymbolValue, (uint*)(workSpace));
}
*maxSymbolValuePtr = 255;
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
}
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
public static nuint HIST_countFast(uint* count, uint* maxSymbolValuePtr, void* source, nuint sourceSize)
{
uint* tmpCounters = stackalloc uint[1024];
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, (void*)tmpCounters, (nuint)(sizeof(uint) * 1024));
}
/*! HIST_count():
* Provides the precise count of each byte within a table 'count'.
* 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
* Updates *maxSymbolValuePtr with actual largest symbol value detected.
* @return : count of the most frequent symbol (which isn't identified).
* or an error code, which can be tested using HIST_isError().
* note : if return == srcSize, there is only one symbol.
*/
public static nuint HIST_count(uint* count, uint* maxSymbolValuePtr, void* src, nuint srcSize)
{
uint* tmpCounters = stackalloc uint[1024];
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, (void*)tmpCounters, (nuint)(sizeof(uint) * 1024));
}
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,225 @@
using System;
using System.Buffers.Binary;
using System.Runtime.CompilerServices;
using InlineIL;
using static InlineIL.IL.Emit;
// ReSharper disable InconsistentNaming
// ReSharper disable IdentifierTypo
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*-**************************************************************
* Memory I/O API
*****************************************************************/
/*=== Static platform detection ===*/
public static bool MEM_32bits => sizeof(nint) == 4;
public static bool MEM_64bits => sizeof(nint) == 8;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
/* default method, safe and standard.
can sometimes prove slower */
[InlineMethod.Inline]
private static ushort MEM_read16(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_U2();
return IL.Return<ushort>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static uint MEM_read32(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_U4();
return IL.Return<uint>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static ulong MEM_read64(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_I8();
return IL.Return<ulong>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static nuint MEM_readST(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_I();
return IL.Return<nuint>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_write64(void* memPtr, ulong value)
{
Ldarg(nameof(memPtr));
Ldarg(nameof(value));
Unaligned(1);
Stind_I8();
}
/*=== Little endian r/w ===*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static ushort MEM_readLE16(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_U2();
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness), typeof(ushort)));
}
return IL.Return<ushort>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_writeLE16(void* memPtr, ushort val)
{
Ldarg(nameof(memPtr));
Ldarg(nameof(val));
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness),
typeof(ushort)));
}
Unaligned(1);
Stind_I2();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static uint MEM_readLE24(void* memPtr) =>
(uint) (MEM_readLE16(memPtr) + (((byte*) memPtr)[2] << 16));
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_writeLE24(void* memPtr, uint val)
{
MEM_writeLE16(memPtr, (ushort) val);
((byte*) memPtr)[2] = (byte) (val >> 16);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static uint MEM_readLE32(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_U4();
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness), typeof(uint)));
}
return IL.Return<uint>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_writeLE32(void* memPtr, uint val32)
{
Ldarg(nameof(memPtr));
Ldarg(nameof(val32));
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness),
typeof(uint)));
}
Unaligned(1);
Stind_I4();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static ulong MEM_readLE64(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_I8();
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness), typeof(ulong)));
}
return IL.Return<ulong>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_writeLE64(void* memPtr, ulong val64)
{
Ldarg(nameof(memPtr));
Ldarg(nameof(val64));
if (!BitConverter.IsLittleEndian)
{
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness),
typeof(ulong)));
}
Unaligned(1);
Stind_I8();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static nuint MEM_readLEST(void* memPtr)
{
Ldarg(nameof(memPtr));
Unaligned(1);
Ldind_I();
if (!BitConverter.IsLittleEndian)
{
Conv_U8();
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness),
typeof(ulong)));
Conv_U();
}
return IL.Return<nuint>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static void MEM_writeLEST(void* memPtr, nuint val)
{
Ldarg(nameof(memPtr));
Ldarg(nameof(val));
if (!BitConverter.IsLittleEndian)
{
Conv_U8();
Call(new MethodRef(typeof(BinaryPrimitives), nameof(BinaryPrimitives.ReverseEndianness),
typeof(ulong)));
Conv_U();
}
Unaligned(1);
Stind_I();
}
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct SeqCollector
{
public int collectSequences;
public ZSTD_Sequence* seqStart;
public nuint seqIndex;
public nuint maxSequences;
}
}

View File

@@ -0,0 +1,17 @@
using System;
namespace ZstdSharp.Unsafe
{
/* **************************
* Canonical representation
****************************/
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
public unsafe partial struct XXH32_canonical_t
{
public fixed byte digest[4];
}
}

View File

@@ -0,0 +1,30 @@
using System;
namespace ZstdSharp.Unsafe
{
/* These definitions are only meant to allow allocation of XXH state
statically, on stack, or in a struct for example.
Do not use members directly. */
public unsafe partial struct XXH32_state_s
{
public uint total_len_32;
public uint large_len;
public uint v1;
public uint v2;
public uint v3;
public uint v4;
/* buffer defined as U32 for alignment */
public fixed uint mem32[4];
public uint memsize;
/* never read nor write, will be removed in a future version */
public uint reserved;
}
}

View File

@@ -0,0 +1,25 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct XXH64_state_s
{
public ulong total_len;
public ulong v1;
public ulong v2;
public ulong v3;
public ulong v4;
/* buffer defined as U64 for alignment */
public fixed ulong mem64[4];
public uint memsize;
/* never read nor write, will be removed in a future version */
public fixed uint reserved[2];
}
}

View File

@@ -0,0 +1,260 @@
using System;
using System.Buffers.Binary;
using System.Numerics;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
// ReSharper disable InconsistentNaming
// ReSharper disable IdentifierTypo
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
private static void XXH_memcpy(void* dest, void* src, ulong size)
{
memcpy((dest), (src), (size));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint XXH_readLE32(void* ptr) =>
BitConverter.IsLittleEndian ? *(uint*) ptr : BinaryPrimitives.ReverseEndianness(*(uint*) ptr);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ulong XXH_readLE64(void* ptr) =>
BitConverter.IsLittleEndian ? *(ulong*) ptr : BinaryPrimitives.ReverseEndianness(*(ulong*) ptr);
public const ulong PRIME64_1 = 11400714785074694791UL;
public const ulong PRIME64_2 = 14029467366897019727UL;
public const ulong PRIME64_3 = 1609587929392839161UL;
public const ulong PRIME64_4 = 9650029242287828579UL;
public const ulong PRIME64_5 = 2870177450012600261UL;
public static uint XXH_versionNumber()
{
return 0 * 100 * 100 + 6 * 100 + 2;
}
private static ulong XXH64_round(ulong acc, ulong input)
{
acc += input * PRIME64_2;
acc = BitOperations.RotateLeft(acc, 31);
acc *= PRIME64_1;
return acc;
}
private static ulong XXH64_mergeRound(ulong acc, ulong val)
{
val = XXH64_round(0, val);
acc ^= val;
acc = acc * PRIME64_1 + PRIME64_4;
return acc;
}
public static void XXH64_reset(XXH64_state_s* statePtr, ulong seed)
{
XXH64_state_s state;
memset(&state, (0), ((ulong)((sizeof(XXH64_state_s) - 8))));
state.v1 = seed + PRIME64_1 + PRIME64_2;
state.v2 = seed + PRIME64_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME64_1;
memcpy(statePtr, &state, ((ulong)sizeof(XXH64_state_s)));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void XXH64_update(XXH64_state_s* state, void* input, ulong len)
{
byte* p = (byte*)(input);
byte* bEnd = p + len;
state->total_len += len;
if (state->memsize + len < 32)
{
if (input != null)
{
XXH_memcpy(((byte*)(state->mem64)) + state->memsize, input, len);
}
state->memsize += (uint)(len);
return;
}
if (state->memsize != 0)
{
XXH_memcpy(((byte*)(state->mem64)) + state->memsize, input, 32 - state->memsize);
state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64 + 0));
state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64 + 1));
state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64 + 2));
state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64 + 3));
p += 32 - state->memsize;
state->memsize = 0;
}
if (p + 32 <= bEnd)
{
byte* limit = bEnd - 32;
ulong v1 = state->v1;
ulong v2 = state->v2;
ulong v3 = state->v3;
ulong v4 = state->v4;
do
{
v1 = XXH64_round(v1, XXH_readLE64(p));
p += 8;
v2 = XXH64_round(v2, XXH_readLE64(p));
p += 8;
v3 = XXH64_round(v3, XXH_readLE64(p));
p += 8;
v4 = XXH64_round(v4, XXH_readLE64(p));
p += 8;
}
while (p <= limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd)
{
XXH_memcpy(state->mem64, p, (ulong)(bEnd - p));
state->memsize = (uint)(bEnd - p);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ulong XXH64_digest(XXH64_state_s* state)
{
byte* p = (byte*)(state->mem64);
byte* bEnd = (byte*)(state->mem64) + state->memsize;
ulong h64;
if (state->total_len >= 32)
{
ulong v1 = state->v1;
ulong v2 = state->v2;
ulong v3 = state->v3;
ulong v4 = state->v4;
h64 = BitOperations.RotateLeft(v1, 1) + BitOperations.RotateLeft(v2, 7) + BitOperations.RotateLeft(v3, 12) + BitOperations.RotateLeft(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
h64 = XXH64_mergeRound(h64, v2);
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
}
else
{
h64 = state->v3 + PRIME64_5;
}
h64 += state->total_len;
while (p + 8 <= bEnd)
{
ulong k1 = XXH64_round(0, XXH_readLE64(p));
h64 ^= k1;
h64 = BitOperations.RotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
p += 8;
}
if (p + 4 <= bEnd)
{
h64 ^= XXH_readLE32(p) * PRIME64_1;
h64 = BitOperations.RotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
p += 4;
}
while (p < bEnd)
{
h64 ^= *p * PRIME64_5;
h64 = BitOperations.RotateLeft(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64>>33;
h64 *= PRIME64_2;
h64 ^= h64>>29;
h64 *= PRIME64_3;
h64 ^= h64>>32;
return h64;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ulong XXH64(void* input, ulong len, ulong seed)
{
byte* p = (byte*)(input);
byte* bEnd = p + len;
ulong h64;
if (len >= ((ulong)(32)))
{
byte* limit = bEnd - 32;
ulong v1 = seed + PRIME64_1 + PRIME64_2;
ulong v2 = seed + PRIME64_2;
ulong v3 = seed + ((ulong)(0));
ulong v4 = seed - PRIME64_1;
do
{
v1 = XXH64_round(v1, XXH_readLE64(((void*)(p))));
p += 8;
v2 = XXH64_round(v2, XXH_readLE64(((void*)(p))));
p += 8;
v3 = XXH64_round(v3, XXH_readLE64(((void*)(p))));
p += 8;
v4 = XXH64_round(v4, XXH_readLE64(((void*)(p))));
p += 8;
}
while (p <= limit);
h64 = BitOperations.RotateLeft(v1, 1) + BitOperations.RotateLeft(v2, 7) + BitOperations.RotateLeft(v3, 12) + BitOperations.RotateLeft(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
h64 = XXH64_mergeRound(h64, v2);
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
}
else
{
h64 = seed + PRIME64_5;
}
h64 += (ulong)(len);
while (p + 8 <= bEnd)
{
ulong k1 = XXH64_round(((ulong)(0)), XXH_readLE64(((void*)(p))));
h64 ^= k1;
h64 = BitOperations.RotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
p += 8;
}
if (p + 4 <= bEnd)
{
h64 ^= (ulong)(XXH_readLE32(((void*)(p)))) * PRIME64_1;
h64 = BitOperations.RotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
p += 4;
}
while (p < bEnd)
{
h64 ^= ((ulong)((*p))) * PRIME64_5;
h64 = BitOperations.RotateLeft(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
}
}

Some files were not shown because too many files have changed in this diff Show More