starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-07-08 18:16:30 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

fix LzmaStream writing async

Browse Source
This commit is contained in:
Adam Hathcock
2026-04-30 13:30:43 +01:00
parent 29f06b753a
commit 6172bd3e52
6 changed files with 1101 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

389
.opencode/package-lock.json generated Normal file
View File

@@ -0,0 +1,389 @@
{
"name": ".opencode",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"@kilocode/plugin": "7.1.20",
"@opencode-ai/plugin": "1.14.30"
}
},
"node_modules/@kilocode/plugin": {
"version": "7.1.20",
"license": "MIT",
"dependencies": {
"@kilocode/sdk": "7.1.20",
"zod": "4.1.8"
}
},
"node_modules/@kilocode/sdk": {
"version": "7.1.20",
"license": "MIT"
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@opencode-ai/plugin": {
"version": "1.14.30",
"resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.14.30.tgz",
"integrity": "sha512-O1y6qR349R5XJtB76Vx4TO/uvLkqNvdsgxtj2ZpWoygb3rtrkuVMiBsrcL2WfuyyaLJnPmbnkeigcdm42r09Hg==",
"license": "MIT",
"dependencies": {
"@opencode-ai/sdk": "1.14.30",
"effect": "4.0.0-beta.57",
"zod": "4.1.8"
},
"peerDependencies": {
"@opentui/core": ">=0.1.105",
"@opentui/solid": ">=0.1.105"
},
"peerDependenciesMeta": {
"@opentui/core": {
"optional": true
},
"@opentui/solid": {
"optional": true
}
}
},
"node_modules/@opencode-ai/sdk": {
"version": "1.14.30",
"resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.14.30.tgz",
"integrity": "sha512-OgPEDvALekHZIjByo/okJ699aLPn+XtsVxgZxUqE8TlzAG7TtskMGFl0fro8O0T2p+nkOT/LstnKGbECvc0+YA==",
"license": "MIT",
"dependencies": {
"cross-spawn": "7.0.6"
}
},
"node_modules/@standard-schema/spec": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==",
"license": "MIT"
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
"which": "^2.0.1"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/detect-libc": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
"integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
"license": "Apache-2.0",
"optional": true,
"engines": {
"node": ">=8"
}
},
"node_modules/effect": {
"version": "4.0.0-beta.57",
"resolved": "https://registry.npmjs.org/effect/-/effect-4.0.0-beta.57.tgz",
"integrity": "sha512-rg32VgXnLKaPRs9tbRDaZ5jxmzNY7ojXt85gSHGUTwdlbWH5Ik+OCUY2q14TXliygPGoHwCAvNWS4bQJOqf00g==",
"license": "MIT",
"dependencies": {
"@standard-schema/spec": "^1.1.0",
"fast-check": "^4.6.0",
"find-my-way-ts": "^0.1.6",
"ini": "^6.0.0",
"kubernetes-types": "^1.30.0",
"msgpackr": "^1.11.9",
"multipasta": "^0.2.7",
"toml": "^4.1.1",
"uuid": "^13.0.0",
"yaml": "^2.8.3"
}
},
"node_modules/fast-check": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.7.0.tgz",
"integrity": "sha512-NsZRtqvSSoCP0HbNjUD+r1JH8zqZalyp6gLY9e7OYs7NK9b6AHOs2baBFeBG7bVNsuoukh89x2Yg3rPsul8ziQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/dubzzz"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fast-check"
}
],
"license": "MIT",
"dependencies": {
"pure-rand": "^8.0.0"
},
"engines": {
"node": ">=12.17.0"
}
},
"node_modules/find-my-way-ts": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/find-my-way-ts/-/find-my-way-ts-0.1.6.tgz",
"integrity": "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==",
"license": "MIT"
},
"node_modules/ini": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz",
"integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==",
"license": "ISC",
"engines": {
"node": "^20.17.0 || >=22.9.0"
}
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
"license": "ISC"
},
"node_modules/kubernetes-types": {
"version": "1.30.0",
"resolved": "https://registry.npmjs.org/kubernetes-types/-/kubernetes-types-1.30.0.tgz",
"integrity": "sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==",
"license": "Apache-2.0"
},
"node_modules/msgpackr": {
"version": "1.11.10",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.10.tgz",
"integrity": "sha512-iCZNq+HszvF+fC3anCm4nBmWEnbeIAfpDs6IStAEKhQ2YSgkjzVG2FF9XJqwwQh5bH3N9OUTUt4QwVN6MLMLtA==",
"license": "MIT",
"optionalDependencies": {
"msgpackr-extract": "^3.0.2"
}
},
"node_modules/msgpackr-extract": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"dependencies": {
"node-gyp-build-optional-packages": "5.2.2"
},
"bin": {
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
},
"optionalDependencies": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
}
},
"node_modules/multipasta": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/multipasta/-/multipasta-0.2.7.tgz",
"integrity": "sha512-KPA58d68KgGil15oDqXjkUBEBYc00XvbPj5/X+dyzeo/lWm9Nc25pQRlf1D+gv4OpK7NM0J1odrbu9JNNGvynA==",
"license": "MIT"
},
"node_modules/node-gyp-build-optional-packages": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
"license": "MIT",
"optional": true,
"dependencies": {
"detect-libc": "^2.0.1"
},
"bin": {
"node-gyp-build-optional-packages": "bin.js",
"node-gyp-build-optional-packages-optional": "optional.js",
"node-gyp-build-optional-packages-test": "build-test.js"
}
},
"node_modules/path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/pure-rand": {
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.0.tgz",
"integrity": "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/dubzzz"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fast-check"
}
],
"license": "MIT"
},
"node_modules/shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
"license": "MIT",
"dependencies": {
"shebang-regex": "^3.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/shebang-regex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/toml": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/toml/-/toml-4.1.1.tgz",
"integrity": "sha512-EBJnVBr3dTXdA89WVFoAIPUqkBjxPMwRqsfuo1r240tKFHXv3zgca4+NJib/h6TyvGF7vOawz0jGuryJCdNHrw==",
"license": "MIT",
"engines": {
"node": ">=20"
}
},
"node_modules/uuid": {
"version": "13.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.1.tgz",
"integrity": "sha512-9ezox2roIft6ExBVTVqibSd5dc5/47Sw/uY6b4SjQUT2TzQ0tltNquWA46y4xPQmdZYqvnio22SgWd41M86+jw==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist-node/bin/uuid"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"license": "ISC",
"dependencies": {
"isexe": "^2.0.0"
},
"bin": {
"node-which": "bin/node-which"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/yaml": {
"version": "2.8.3",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
"integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
"license": "ISC",
"bin": {
"yaml": "bin.mjs"
},
"engines": {
"node": ">= 14.6"
},
"funding": {
"url": "https://github.com/sponsors/eemeli"
}
},
"node_modules/zod": {
"version": "4.1.8",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
}
}
}

2
src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs
View File

@@ -122,7 +122,7 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream)
uint inputCrc2;
long inputSize2;
{
using var lzma2Stream = new Lzma2EncoderStream(
await using var lzma2Stream = new Lzma2EncoderStream(
outCrcStream,
encoderProperties.DictionarySize,
encoderProperties.NumFastBytes

178
src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs
View File

@@ -1,5 +1,7 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace SharpCompress.Compressors.LZMA;
@@ -77,6 +79,56 @@ internal sealed class Lzma2EncoderStream : Stream
}
}
public override async Task WriteAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
while (count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var toCopy = Math.Min(count, _buffer.Length - _bufferPosition);
Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy);
_bufferPosition += toCopy;
offset += toCopy;
count -= toCopy;
if (_bufferPosition == _buffer.Length)
{
await FlushChunkAsync(cancellationToken).ConfigureAwait(false);
}
}
}
#if !LEGACY_DOTNET
public override async ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default
)
{
var offset = 0;
var count = buffer.Length;
while (count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var toCopy = Math.Min(count, _buffer.Length - _bufferPosition);
buffer.Slice(offset, toCopy).Span.CopyTo(_buffer.AsSpan(_bufferPosition, toCopy));
_bufferPosition += toCopy;
offset += toCopy;
count -= toCopy;
if (_bufferPosition == _buffer.Length)
{
await FlushChunkAsync(cancellationToken).ConfigureAwait(false);
}
}
}
#endif
public override void Flush() { }
public override int Read(byte[] buffer, int offset, int count) =>
@@ -104,6 +156,29 @@ internal sealed class Lzma2EncoderStream : Stream
base.Dispose(disposing);
}
#if !LEGACY_DOTNET || NETSTANDARD2_1
public override async ValueTask DisposeAsync()
#else
public async ValueTask DisposeAsync()
#endif
{
if (!_isDisposed)
{
_isDisposed = true;
if (_bufferPosition > 0)
{
await FlushChunkAsync().ConfigureAwait(false);
}
await _output.WriteAsync(new byte[] { 0x00 }, 0, 1).ConfigureAwait(false);
}
#if !LEGACY_DOTNET || NETSTANDARD2_1
await base.DisposeAsync().ConfigureAwait(false);
#endif
}
private void FlushChunk()
{
if (_bufferPosition == 0)
@@ -141,6 +216,43 @@ internal sealed class Lzma2EncoderStream : Stream
}
}
private async ValueTask FlushChunkAsync(CancellationToken cancellationToken = default)
{
if (_bufferPosition == 0)
{
return;
}
var uncompressedData = _buffer.AsMemory(0, _bufferPosition);
_bufferPosition = 0;
byte[] compressed;
try
{
compressed = CompressBlock(uncompressedData.Span);
}
catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException)
{
await WriteUncompressedChunksAsync(uncompressedData, cancellationToken)
.ConfigureAwait(false);
return;
}
if (
compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE
&& compressed.Length < uncompressedData.Length
)
{
await WriteCompressedChunkAsync(uncompressedData.Length, compressed, cancellationToken)
.ConfigureAwait(false);
}
else
{
await WriteUncompressedChunksAsync(uncompressedData, cancellationToken)
.ConfigureAwait(false);
}
}
private byte[] CompressBlock(ReadOnlySpan<byte> data)
{
var encoderProps = new LzmaEncoderProperties(eos: false, _dictionarySize, _numFastBytes);
@@ -225,6 +337,32 @@ internal sealed class Lzma2EncoderStream : Stream
_output.Write(compressedData, 0, compressedData.Length);
}
private async ValueTask WriteCompressedChunkAsync(
int uncompressedSize,
byte[] compressedData,
CancellationToken cancellationToken = default
)
{
var uncompSizeMinus1 = uncompressedSize - 1;
var compSizeMinus1 = compressedData.Length - 1;
var control = (byte)(0xE0 | ((uncompSizeMinus1 >> 16) & 0x1F));
_isFirstChunk = false;
var header = new[]
{
control,
(byte)((uncompSizeMinus1 >> 8) & 0xFF),
(byte)(uncompSizeMinus1 & 0xFF),
(byte)((compSizeMinus1 >> 8) & 0xFF),
(byte)(compSizeMinus1 & 0xFF),
_lzmaPropertiesByte,
};
await _output.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false);
await _output
.WriteAsync(compressedData, 0, compressedData.Length, cancellationToken)
.ConfigureAwait(false);
}
/// <summary>
/// Writes data as uncompressed LZMA2 sub-chunks (max 64KB each).
/// Header: [control] [size_hi] [size_lo]
@@ -259,6 +397,46 @@ internal sealed class Lzma2EncoderStream : Stream
}
}
private async ValueTask WriteUncompressedChunksAsync(
ReadOnlyMemory<byte> data,
CancellationToken cancellationToken = default
)
{
var offset = 0;
while (offset < data.Length)
{
var chunkSize = Math.Min(data.Length - offset, MAX_UNCOMPRESSED_SUBCHUNK_SIZE);
var sizeMinus1 = chunkSize - 1;
byte control;
if (_isFirstChunk)
{
control = 0x01;
_isFirstChunk = false;
}
else
{
control = 0x02;
}
var header = new[]
{
control,
(byte)((sizeMinus1 >> 8) & 0xFF),
(byte)(sizeMinus1 & 0xFF),
};
await _output
.WriteAsync(header, 0, header.Length, cancellationToken)
.ConfigureAwait(false);
var chunk = data.Slice(offset, chunkSize).ToArray();
await _output
.WriteAsync(chunk, 0, chunk.Length, cancellationToken)
.ConfigureAwait(false);
offset += chunkSize;
}
}
/// <summary>
/// Encodes a dictionary size into the 1-byte LZMA2 properties format.
/// Reverse of the decoder formula: dictSize = (2 | (p and 1)) shl ((p shr 1) + 11)

478
src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs
View File

@@ -2,13 +2,15 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Compressors.LZMA.LZ;
using SharpCompress.Compressors.LZMA.RangeCoder;
namespace SharpCompress.Compressors.LZMA;
internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
{
private enum EMatchFinderType
{
@@ -79,9 +81,9 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
private const int K_DEFAULT_DICTIONARY_LOG_SIZE = 22;
private const uint K_NUM_FAST_BYTES_DEFAULT = 0x20;
private class LiteralEncoder
private partial class LiteralEncoder
{
public struct Encoder2
public partial struct Encoder2
{
private BitEncoder[] _encoders;
@@ -106,6 +108,23 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
public async ValueTask EncodeAsync(
RangeCoder.Encoder rangeEncoder,
byte symbol,
CancellationToken cancellationToken = default
)
{
uint context = 1;
for (var i = 7; i >= 0; i--)
{
var bit = (uint)((symbol >> i) & 1);
await _encoders[context]
.EncodeAsync(rangeEncoder, bit, cancellationToken)
.ConfigureAwait(false);
context = (context << 1) | bit;
}
}
public void EncodeMatched(RangeCoder.Encoder rangeEncoder, byte matchByte, byte symbol)
{
uint context = 1;
@@ -125,6 +144,32 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
public async ValueTask EncodeMatchedAsync(
RangeCoder.Encoder rangeEncoder,
byte matchByte,
byte symbol,
CancellationToken cancellationToken = default
)
{
uint context = 1;
var same = true;
for (var i = 7; i >= 0; i--)
{
var bit = (uint)((symbol >> i) & 1);
var state = context;
if (same)
{
var matchBit = (uint)((matchByte >> i) & 1);
state += ((1 + matchBit) << 8);
same = matchBit == bit;
}
await _encoders[state]
.EncodeAsync(rangeEncoder, bit, cancellationToken)
.ConfigureAwait(false);
context = (context << 1) | bit;
}
}
public uint GetPrice(bool matchMode, byte matchByte, byte symbol)
{
uint price = 0;
@@ -190,7 +235,7 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
_coders[((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits))];
}
private class LenEncoder
private partial class LenEncoder
{
private BitEncoder _choice = new();
private BitEncoder _choice2 = new();
@@ -247,6 +292,49 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
public async ValueTask EncodeAsync(
RangeCoder.Encoder rangeEncoder,
uint symbol,
uint posState,
CancellationToken cancellationToken = default
)
{
if (symbol < Base.K_NUM_LOW_LEN_SYMBOLS)
{
await _choice.EncodeAsync(rangeEncoder, 0, cancellationToken).ConfigureAwait(false);
await _lowCoder[posState]
.EncodeAsync(rangeEncoder, symbol, cancellationToken)
.ConfigureAwait(false);
}
else
{
symbol -= Base.K_NUM_LOW_LEN_SYMBOLS;
await _choice.EncodeAsync(rangeEncoder, 1, cancellationToken).ConfigureAwait(false);
if (symbol < Base.K_NUM_MID_LEN_SYMBOLS)
{
await _choice2
.EncodeAsync(rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
await _midCoder[posState]
.EncodeAsync(rangeEncoder, symbol, cancellationToken)
.ConfigureAwait(false);
}
else
{
await _choice2
.EncodeAsync(rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
await _highCoder
.EncodeAsync(
rangeEncoder,
symbol - Base.K_NUM_MID_LEN_SYMBOLS,
cancellationToken
)
.ConfigureAwait(false);
}
}
}
public void SetPrices(uint posState, uint numSymbols, uint[] prices, uint st)
{
var a0 = _choice.GetPrice0();
@@ -281,7 +369,7 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
private class LenPriceTableEncoder : LenEncoder
private partial class LenPriceTableEncoder : LenEncoder
{
private readonly uint[] _prices = new uint[
Base.K_NUM_LEN_SYMBOLS << Base.K_NUM_POS_STATES_BITS_ENCODING_MAX
@@ -316,6 +404,21 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
UpdateTable(posState);
}
}
public new async ValueTask EncodeAsync(
RangeCoder.Encoder rangeEncoder,
uint symbol,
uint posState,
CancellationToken cancellationToken = default
)
{
await base.EncodeAsync(rangeEncoder, symbol, posState, cancellationToken)
.ConfigureAwait(false);
if (--_counters[posState] == 0)
{
UpdateTable(posState);
}
}
}
private const uint K_NUM_OPTS = 1 << 12;
@@ -1254,6 +1357,46 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
_posAlignEncoder.ReverseEncode(_rangeEncoder, posReduced & Base.K_ALIGN_MASK);
}
private async ValueTask WriteEndMarkerAsync(
uint posState,
CancellationToken cancellationToken = default
)
{
if (!_writeEndMark)
{
return;
}
await _isMatch[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState]
.EncodeAsync(_rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
await _isRep[_state._index]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
_state.UpdateMatch();
var len = Base.K_MATCH_MIN_LEN;
await _lenEncoder
.EncodeAsync(_rangeEncoder, len - Base.K_MATCH_MIN_LEN, posState, cancellationToken)
.ConfigureAwait(false);
uint posSlot = (1 << Base.K_NUM_POS_SLOT_BITS) - 1;
var lenToPosState = Base.GetLenToPosState(len);
await _posSlotEncoder[lenToPosState]
.EncodeAsync(_rangeEncoder, posSlot, cancellationToken)
.ConfigureAwait(false);
var footerBits = 30;
var posReduced = (((uint)1) << footerBits) - 1;
await _rangeEncoder
.EncodeDirectBitsAsync(
posReduced >> Base.K_NUM_ALIGN_BITS,
footerBits - Base.K_NUM_ALIGN_BITS,
cancellationToken
)
.ConfigureAwait(false);
await _posAlignEncoder
.ReverseEncodeAsync(_rangeEncoder, posReduced & Base.K_ALIGN_MASK, cancellationToken)
.ConfigureAwait(false);
}
private void Flush(uint nowPos)
{
ReleaseMfStream();
@@ -1262,6 +1405,17 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
_rangeEncoder.FlushStream();
}
private async ValueTask FlushAsync(uint nowPos, CancellationToken cancellationToken = default)
{
ReleaseMfStream();
await WriteEndMarkerAsync(nowPos & _posStateMask, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < 5; i++)
{
await _rangeEncoder.ShiftLowAsync(cancellationToken).ConfigureAwait(false);
}
await _rangeEncoder.FlushStreamAsync(cancellationToken).ConfigureAwait(false);
}
public void CodeOneBlock(out long inSize, out long outSize, out bool finished)
{
inSize = 0;
@@ -1503,6 +1657,290 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
public async ValueTask<(long inSize, long outSize, bool finished)> CodeOneBlockAsync(
CancellationToken cancellationToken = default
)
{
long inSize = 0;
long outSize = 0;
var finished = true;
if (_inStream != null)
{
_matchFinder.SetStream(_inStream);
_needReleaseMfStream = true;
_inStream = null;
}
if (_finished)
{
return (inSize, outSize, finished);
}
_finished = true;
var progressPosValuePrev = _nowPos64;
if (_nowPos64 == 0)
{
if (_trainSize > 0)
{
for (
;
_trainSize > 0 && (!_processingMode || !_matchFinder.IsDataStarved);
_trainSize--
)
{
_matchFinder.Skip(1);
}
if (_trainSize == 0)
{
_previousByte = _matchFinder.GetIndexByte(-1);
}
}
if (_processingMode && _matchFinder.IsDataStarved)
{
_finished = false;
return (inSize, outSize, finished);
}
if (_matchFinder.GetNumAvailableBytes() == 0)
{
await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false);
return (inSize, outSize, finished);
}
ReadMatchDistances(out var len, out var numDistancePairs);
var posState = (uint)_nowPos64 & _posStateMask;
await _isMatch[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
_state.UpdateChar();
var curByte = _matchFinder.GetIndexByte((int)(0 - _additionalOffset));
await _literalEncoder
.GetSubCoder((uint)_nowPos64, _previousByte)
.EncodeAsync(_rangeEncoder, curByte, cancellationToken)
.ConfigureAwait(false);
_previousByte = curByte;
_additionalOffset--;
_nowPos64++;
}
if (_processingMode && _matchFinder.IsDataStarved)
{
_finished = false;
return (inSize, outSize, finished);
}
if (_matchFinder.GetNumAvailableBytes() == 0)
{
await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false);
return (inSize, outSize, finished);
}
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
if (_processingMode && _matchFinder.IsDataStarved)
{
_finished = false;
return (inSize, outSize, finished);
}
var len = GetOptimum((uint)_nowPos64, out var pos);
var posState = (uint)_nowPos64 & _posStateMask;
var complexState = (_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState;
if (len == 1 && pos == 0xFFFFFFFF)
{
await _isMatch[complexState]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
var curByte = _matchFinder.GetIndexByte((int)(0 - _additionalOffset));
var subCoder = _literalEncoder.GetSubCoder((uint)_nowPos64, _previousByte);
if (!_state.IsCharState())
{
var matchByte = _matchFinder.GetIndexByte(
(int)(0 - _repDistances[0] - 1 - _additionalOffset)
);
await subCoder
.EncodeMatchedAsync(_rangeEncoder, matchByte, curByte, cancellationToken)
.ConfigureAwait(false);
}
else
{
await subCoder
.EncodeAsync(_rangeEncoder, curByte, cancellationToken)
.ConfigureAwait(false);
}
_previousByte = curByte;
_state.UpdateChar();
}
else
{
await _isMatch[complexState]
.EncodeAsync(_rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
if (pos < Base.K_NUM_REP_DISTANCES)
{
await _isRep[_state._index]
.EncodeAsync(_rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
if (pos == 0)
{
await _isRepG0[_state._index]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
await _isRep0Long[complexState]
.EncodeAsync(_rangeEncoder, len == 1 ? 0u : 1u, cancellationToken)
.ConfigureAwait(false);
}
else
{
await _isRepG0[_state._index]
.EncodeAsync(_rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
if (pos == 1)
{
await _isRepG1[_state._index]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
}
else
{
await _isRepG1[_state._index]
.EncodeAsync(_rangeEncoder, 1, cancellationToken)
.ConfigureAwait(false);
await _isRepG2[_state._index]
.EncodeAsync(_rangeEncoder, pos - 2, cancellationToken)
.ConfigureAwait(false);
}
}
if (len == 1)
{
_state.UpdateShortRep();
}
else
{
await _repMatchLenEncoder
.EncodeAsync(
_rangeEncoder,
len - Base.K_MATCH_MIN_LEN,
posState,
cancellationToken
)
.ConfigureAwait(false);
_state.UpdateRep();
}
var distance = _repDistances[pos];
if (pos != 0)
{
for (var i = pos; i >= 1; i--)
{
_repDistances[i] = _repDistances[i - 1];
}
_repDistances[0] = distance;
}
}
else
{
await _isRep[_state._index]
.EncodeAsync(_rangeEncoder, 0, cancellationToken)
.ConfigureAwait(false);
_state.UpdateMatch();
await _lenEncoder
.EncodeAsync(
_rangeEncoder,
len - Base.K_MATCH_MIN_LEN,
posState,
cancellationToken
)
.ConfigureAwait(false);
pos -= Base.K_NUM_REP_DISTANCES;
var posSlot = GetPosSlot(pos);
var lenToPosState = Base.GetLenToPosState(len);
await _posSlotEncoder[lenToPosState]
.EncodeAsync(_rangeEncoder, posSlot, cancellationToken)
.ConfigureAwait(false);
if (posSlot >= Base.K_START_POS_MODEL_INDEX)
{
var footerBits = (int)((posSlot >> 1) - 1);
var baseVal = ((2 | (posSlot & 1)) << footerBits);
var posReduced = pos - baseVal;
if (posSlot < Base.K_END_POS_MODEL_INDEX)
{
await BitTreeEncoder
.ReverseEncodeAsync(
_posEncoders,
baseVal - posSlot - 1,
_rangeEncoder,
footerBits,
posReduced,
cancellationToken
)
.ConfigureAwait(false);
}
else
{
await _rangeEncoder
.EncodeDirectBitsAsync(
posReduced >> Base.K_NUM_ALIGN_BITS,
footerBits - Base.K_NUM_ALIGN_BITS,
cancellationToken
)
.ConfigureAwait(false);
await _posAlignEncoder
.ReverseEncodeAsync(
_rangeEncoder,
posReduced & Base.K_ALIGN_MASK,
cancellationToken
)
.ConfigureAwait(false);
_alignPriceCount++;
}
}
var distance = pos;
for (var i = Base.K_NUM_REP_DISTANCES - 1; i >= 1; i--)
{
_repDistances[i] = _repDistances[i - 1];
}
_repDistances[0] = distance;
_matchPriceCount++;
}
_previousByte = _matchFinder.GetIndexByte((int)(len - 1 - _additionalOffset));
}
_additionalOffset -= len;
_nowPos64 += len;
if (_additionalOffset == 0)
{
if (_matchPriceCount >= (1 << 7))
{
FillDistancesPrices();
}
if (_alignPriceCount >= Base.K_ALIGN_TABLE_SIZE)
{
FillAlignPrices();
}
inSize = _nowPos64;
outSize = _rangeEncoder.GetProcessedSizeAdd();
if (_processingMode && _matchFinder.IsDataStarved)
{
_finished = false;
return (inSize, outSize, finished);
}
if (_matchFinder.GetNumAvailableBytes() == 0)
{
await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false);
return (inSize, outSize, finished);
}
if (_nowPos64 - progressPosValuePrev >= (1 << 12))
{
_finished = false;
finished = false;
return (inSize, outSize, finished);
}
}
}
}
private void ReleaseMfStream()
{
if (_matchFinder != null && _needReleaseMfStream)
@@ -1599,6 +2037,36 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties
}
}
public async ValueTask<long> CodeAsync(
Stream inStream,
bool final,
CancellationToken cancellationToken = default
)
{
_matchFinder.SetStream(inStream);
_processingMode = !final;
try
{
while (true)
{
var (processedInSize, _, finished) = await CodeOneBlockAsync(cancellationToken)
.ConfigureAwait(false);
if (finished)
{
return processedInSize;
}
}
}
finally
{
_matchFinder.ReleaseStream();
if (final)
{
ReleaseStreams();
}
}
}
public void Train(Stream trainStream)
{
if (_nowPos64 > 0)

62
src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs
View File

@@ -389,7 +389,7 @@ public partial class LzmaStream
}
#endif
public override Task WriteAsync(
public override async Task WriteAsync(
byte[] buffer,
int offset,
int count,
@@ -397,7 +397,63 @@ public partial class LzmaStream
)
{
cancellationToken.ThrowIfCancellationRequested();
Write(buffer, offset, count);
return Task.CompletedTask;
if (_encoder != null)
{
_position = await _encoder
.CodeAsync(new MemoryStream(buffer, offset, count), false, cancellationToken)
.ConfigureAwait(false);
}
}
#if !LEGACY_DOTNET
public override async ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
if (_encoder != null)
{
_position = await _encoder
.CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken)
.ConfigureAwait(false);
}
}
#endif
#if !LEGACY_DOTNET || NETSTANDARD2_1
public override async ValueTask DisposeAsync()
#else
public async ValueTask DisposeAsync()
#endif
{
if (_isDisposed)
{
return;
}
_isDisposed = true;
if (_encoder != null)
{
_position = await _encoder.CodeAsync(null, true).ConfigureAwait(false);
}
if (!_leaveOpen)
{
if (_inputStream is IAsyncDisposable asyncDisposableInputStream)
{
await asyncDisposableInputStream.DisposeAsync().ConfigureAwait(false);
}
else
{
_inputStream?.Dispose();
}
}
await _outWindow.DisposeAsync().ConfigureAwait(false);
#if !LEGACY_DOTNET || NETSTANDARD2_1
await base.DisposeAsync().ConfigureAwait(false);
#endif
GC.SuppressFinalize(this);
}
}

2
src/SharpCompress/Compressors/LZMA/LzmaStream.cs
View File

@@ -8,7 +8,7 @@ using SharpCompress.IO;
namespace SharpCompress.Compressors.LZMA;
public partial class LzmaStream : Stream, IStreamStack
public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable
{
private readonly Stream? _inputStream;
private readonly long _inputSize;