diff --git a/.opencode/package-lock.json b/.opencode/package-lock.json new file mode 100644 index 00000000..b1b08b4a --- /dev/null +++ b/.opencode/package-lock.json @@ -0,0 +1,389 @@ +{ + "name": ".opencode", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@kilocode/plugin": "7.1.20", + "@opencode-ai/plugin": "1.14.30" + } + }, + "node_modules/@kilocode/plugin": { + "version": "7.1.20", + "license": "MIT", + "dependencies": { + "@kilocode/sdk": "7.1.20", + "zod": "4.1.8" + } + }, + "node_modules/@kilocode/sdk": { + "version": "7.1.20", + "license": "MIT" + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@opencode-ai/plugin": { + "version": "1.14.30", + "resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.14.30.tgz", + "integrity": "sha512-O1y6qR349R5XJtB76Vx4TO/uvLkqNvdsgxtj2ZpWoygb3rtrkuVMiBsrcL2WfuyyaLJnPmbnkeigcdm42r09Hg==", + "license": "MIT", + "dependencies": { + "@opencode-ai/sdk": "1.14.30", + "effect": "4.0.0-beta.57", + "zod": "4.1.8" + }, + "peerDependencies": { + "@opentui/core": ">=0.1.105", + "@opentui/solid": ">=0.1.105" + }, + "peerDependenciesMeta": { + "@opentui/core": { + "optional": true + }, + "@opentui/solid": { + "optional": true + } + } + }, + "node_modules/@opencode-ai/sdk": { + "version": "1.14.30", + "resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.14.30.tgz", + "integrity": "sha512-OgPEDvALekHZIjByo/okJ699aLPn+XtsVxgZxUqE8TlzAG7TtskMGFl0fro8O0T2p+nkOT/LstnKGbECvc0+YA==", + "license": "MIT", + "dependencies": { + "cross-spawn": "7.0.6" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/effect": { + "version": "4.0.0-beta.57", + "resolved": "https://registry.npmjs.org/effect/-/effect-4.0.0-beta.57.tgz", + "integrity": "sha512-rg32VgXnLKaPRs9tbRDaZ5jxmzNY7ojXt85gSHGUTwdlbWH5Ik+OCUY2q14TXliygPGoHwCAvNWS4bQJOqf00g==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "fast-check": "^4.6.0", + "find-my-way-ts": "^0.1.6", + "ini": "^6.0.0", + "kubernetes-types": "^1.30.0", + "msgpackr": "^1.11.9", + "multipasta": "^0.2.7", + "toml": "^4.1.1", + "uuid": "^13.0.0", + "yaml": "^2.8.3" + } + }, + "node_modules/fast-check": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.7.0.tgz", + "integrity": "sha512-NsZRtqvSSoCP0HbNjUD+r1JH8zqZalyp6gLY9e7OYs7NK9b6AHOs2baBFeBG7bVNsuoukh89x2Yg3rPsul8ziQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, + "node_modules/find-my-way-ts": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/find-my-way-ts/-/find-my-way-ts-0.1.6.tgz", + "integrity": "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==", + "license": "MIT" + }, + "node_modules/ini": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", + "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/kubernetes-types": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/kubernetes-types/-/kubernetes-types-1.30.0.tgz", + "integrity": "sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==", + "license": "Apache-2.0" + }, + "node_modules/msgpackr": { + "version": "1.11.10", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.10.tgz", + "integrity": "sha512-iCZNq+HszvF+fC3anCm4nBmWEnbeIAfpDs6IStAEKhQ2YSgkjzVG2FF9XJqwwQh5bH3N9OUTUt4QwVN6MLMLtA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, + "node_modules/multipasta": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/multipasta/-/multipasta-0.2.7.tgz", + "integrity": "sha512-KPA58d68KgGil15oDqXjkUBEBYc00XvbPj5/X+dyzeo/lWm9Nc25pQRlf1D+gv4OpK7NM0J1odrbu9JNNGvynA==", + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pure-rand": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.0.tgz", + "integrity": "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/toml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/toml/-/toml-4.1.1.tgz", + "integrity": "sha512-EBJnVBr3dTXdA89WVFoAIPUqkBjxPMwRqsfuo1r240tKFHXv3zgca4+NJib/h6TyvGF7vOawz0jGuryJCdNHrw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/uuid": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.1.tgz", + "integrity": "sha512-9ezox2roIft6ExBVTVqibSd5dc5/47Sw/uY6b4SjQUT2TzQ0tltNquWA46y4xPQmdZYqvnio22SgWd41M86+jw==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.1.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs b/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs index fa5f1a0d..5c8aedc4 100644 --- a/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs +++ b/src/SharpCompress/Common/SevenZip/SevenZipStreamsCompressor.cs @@ -122,7 +122,7 @@ internal sealed class SevenZipStreamsCompressor(Stream outputStream) uint inputCrc2; long inputSize2; { - using var lzma2Stream = new Lzma2EncoderStream( + await using var lzma2Stream = new Lzma2EncoderStream( outCrcStream, encoderProperties.DictionarySize, encoderProperties.NumFastBytes diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index 1642be44..73e3c7af 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; namespace SharpCompress.Compressors.LZMA; @@ -77,6 +79,56 @@ internal sealed class Lzma2EncoderStream : Stream } } + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + while (count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + + var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy); + _bufferPosition += toCopy; + offset += toCopy; + count -= toCopy; + + if (_bufferPosition == _buffer.Length) + { + await FlushChunkAsync(cancellationToken).ConfigureAwait(false); + } + } + } + +#if !LEGACY_DOTNET + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + var offset = 0; + var count = buffer.Length; + while (count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + + var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + buffer.Slice(offset, toCopy).Span.CopyTo(_buffer.AsSpan(_bufferPosition, toCopy)); + _bufferPosition += toCopy; + offset += toCopy; + count -= toCopy; + + if (_bufferPosition == _buffer.Length) + { + await FlushChunkAsync(cancellationToken).ConfigureAwait(false); + } + } + } +#endif + public override void Flush() { } public override int Read(byte[] buffer, int offset, int count) => @@ -104,6 +156,29 @@ internal sealed class Lzma2EncoderStream : Stream base.Dispose(disposing); } +#if !LEGACY_DOTNET || NETSTANDARD2_1 + public override async ValueTask DisposeAsync() +#else + public async ValueTask DisposeAsync() +#endif + { + if (!_isDisposed) + { + _isDisposed = true; + + if (_bufferPosition > 0) + { + await FlushChunkAsync().ConfigureAwait(false); + } + + await _output.WriteAsync(new byte[] { 0x00 }, 0, 1).ConfigureAwait(false); + } + +#if !LEGACY_DOTNET || NETSTANDARD2_1 + await base.DisposeAsync().ConfigureAwait(false); +#endif + } + private void FlushChunk() { if (_bufferPosition == 0) @@ -141,6 +216,43 @@ internal sealed class Lzma2EncoderStream : Stream } } + private async ValueTask FlushChunkAsync(CancellationToken cancellationToken = default) + { + if (_bufferPosition == 0) + { + return; + } + + var uncompressedData = _buffer.AsMemory(0, _bufferPosition); + _bufferPosition = 0; + + byte[] compressed; + try + { + compressed = CompressBlock(uncompressedData.Span); + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + await WriteUncompressedChunksAsync(uncompressedData, cancellationToken) + .ConfigureAwait(false); + return; + } + + if ( + compressed.Length <= MAX_COMPRESSED_CHUNK_SIZE + && compressed.Length < uncompressedData.Length + ) + { + await WriteCompressedChunkAsync(uncompressedData.Length, compressed, cancellationToken) + .ConfigureAwait(false); + } + else + { + await WriteUncompressedChunksAsync(uncompressedData, cancellationToken) + .ConfigureAwait(false); + } + } + private byte[] CompressBlock(ReadOnlySpan data) { var encoderProps = new LzmaEncoderProperties(eos: false, _dictionarySize, _numFastBytes); @@ -225,6 +337,32 @@ internal sealed class Lzma2EncoderStream : Stream _output.Write(compressedData, 0, compressedData.Length); } + private async ValueTask WriteCompressedChunkAsync( + int uncompressedSize, + byte[] compressedData, + CancellationToken cancellationToken = default + ) + { + var uncompSizeMinus1 = uncompressedSize - 1; + var compSizeMinus1 = compressedData.Length - 1; + var control = (byte)(0xE0 | ((uncompSizeMinus1 >> 16) & 0x1F)); + _isFirstChunk = false; + + var header = new[] + { + control, + (byte)((uncompSizeMinus1 >> 8) & 0xFF), + (byte)(uncompSizeMinus1 & 0xFF), + (byte)((compSizeMinus1 >> 8) & 0xFF), + (byte)(compSizeMinus1 & 0xFF), + _lzmaPropertiesByte, + }; + await _output.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false); + await _output + .WriteAsync(compressedData, 0, compressedData.Length, cancellationToken) + .ConfigureAwait(false); + } + /// /// Writes data as uncompressed LZMA2 sub-chunks (max 64KB each). /// Header: [control] [size_hi] [size_lo] @@ -259,6 +397,46 @@ internal sealed class Lzma2EncoderStream : Stream } } + private async ValueTask WriteUncompressedChunksAsync( + ReadOnlyMemory data, + CancellationToken cancellationToken = default + ) + { + var offset = 0; + while (offset < data.Length) + { + var chunkSize = Math.Min(data.Length - offset, MAX_UNCOMPRESSED_SUBCHUNK_SIZE); + var sizeMinus1 = chunkSize - 1; + + byte control; + if (_isFirstChunk) + { + control = 0x01; + _isFirstChunk = false; + } + else + { + control = 0x02; + } + + var header = new[] + { + control, + (byte)((sizeMinus1 >> 8) & 0xFF), + (byte)(sizeMinus1 & 0xFF), + }; + await _output + .WriteAsync(header, 0, header.Length, cancellationToken) + .ConfigureAwait(false); + + var chunk = data.Slice(offset, chunkSize).ToArray(); + await _output + .WriteAsync(chunk, 0, chunk.Length, cancellationToken) + .ConfigureAwait(false); + offset += chunkSize; + } + } + /// /// Encodes a dictionary size into the 1-byte LZMA2 properties format. /// Reverse of the decoder formula: dictSize = (2 | (p and 1)) shl ((p shr 1) + 11) diff --git a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs index d674cf45..14d94707 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs @@ -2,13 +2,15 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Compressors.LZMA.LZ; using SharpCompress.Compressors.LZMA.RangeCoder; namespace SharpCompress.Compressors.LZMA; -internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties +internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties { private enum EMatchFinderType { @@ -79,9 +81,9 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties private const int K_DEFAULT_DICTIONARY_LOG_SIZE = 22; private const uint K_NUM_FAST_BYTES_DEFAULT = 0x20; - private class LiteralEncoder + private partial class LiteralEncoder { - public struct Encoder2 + public partial struct Encoder2 { private BitEncoder[] _encoders; @@ -106,6 +108,23 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } + public async ValueTask EncodeAsync( + RangeCoder.Encoder rangeEncoder, + byte symbol, + CancellationToken cancellationToken = default + ) + { + uint context = 1; + for (var i = 7; i >= 0; i--) + { + var bit = (uint)((symbol >> i) & 1); + await _encoders[context] + .EncodeAsync(rangeEncoder, bit, cancellationToken) + .ConfigureAwait(false); + context = (context << 1) | bit; + } + } + public void EncodeMatched(RangeCoder.Encoder rangeEncoder, byte matchByte, byte symbol) { uint context = 1; @@ -125,6 +144,32 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } + public async ValueTask EncodeMatchedAsync( + RangeCoder.Encoder rangeEncoder, + byte matchByte, + byte symbol, + CancellationToken cancellationToken = default + ) + { + uint context = 1; + var same = true; + for (var i = 7; i >= 0; i--) + { + var bit = (uint)((symbol >> i) & 1); + var state = context; + if (same) + { + var matchBit = (uint)((matchByte >> i) & 1); + state += ((1 + matchBit) << 8); + same = matchBit == bit; + } + await _encoders[state] + .EncodeAsync(rangeEncoder, bit, cancellationToken) + .ConfigureAwait(false); + context = (context << 1) | bit; + } + } + public uint GetPrice(bool matchMode, byte matchByte, byte symbol) { uint price = 0; @@ -190,7 +235,7 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties _coders[((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits))]; } - private class LenEncoder + private partial class LenEncoder { private BitEncoder _choice = new(); private BitEncoder _choice2 = new(); @@ -247,6 +292,49 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } + public async ValueTask EncodeAsync( + RangeCoder.Encoder rangeEncoder, + uint symbol, + uint posState, + CancellationToken cancellationToken = default + ) + { + if (symbol < Base.K_NUM_LOW_LEN_SYMBOLS) + { + await _choice.EncodeAsync(rangeEncoder, 0, cancellationToken).ConfigureAwait(false); + await _lowCoder[posState] + .EncodeAsync(rangeEncoder, symbol, cancellationToken) + .ConfigureAwait(false); + } + else + { + symbol -= Base.K_NUM_LOW_LEN_SYMBOLS; + await _choice.EncodeAsync(rangeEncoder, 1, cancellationToken).ConfigureAwait(false); + if (symbol < Base.K_NUM_MID_LEN_SYMBOLS) + { + await _choice2 + .EncodeAsync(rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + await _midCoder[posState] + .EncodeAsync(rangeEncoder, symbol, cancellationToken) + .ConfigureAwait(false); + } + else + { + await _choice2 + .EncodeAsync(rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + await _highCoder + .EncodeAsync( + rangeEncoder, + symbol - Base.K_NUM_MID_LEN_SYMBOLS, + cancellationToken + ) + .ConfigureAwait(false); + } + } + } + public void SetPrices(uint posState, uint numSymbols, uint[] prices, uint st) { var a0 = _choice.GetPrice0(); @@ -281,7 +369,7 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } - private class LenPriceTableEncoder : LenEncoder + private partial class LenPriceTableEncoder : LenEncoder { private readonly uint[] _prices = new uint[ Base.K_NUM_LEN_SYMBOLS << Base.K_NUM_POS_STATES_BITS_ENCODING_MAX @@ -316,6 +404,21 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties UpdateTable(posState); } } + + public new async ValueTask EncodeAsync( + RangeCoder.Encoder rangeEncoder, + uint symbol, + uint posState, + CancellationToken cancellationToken = default + ) + { + await base.EncodeAsync(rangeEncoder, symbol, posState, cancellationToken) + .ConfigureAwait(false); + if (--_counters[posState] == 0) + { + UpdateTable(posState); + } + } } private const uint K_NUM_OPTS = 1 << 12; @@ -1254,6 +1357,46 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties _posAlignEncoder.ReverseEncode(_rangeEncoder, posReduced & Base.K_ALIGN_MASK); } + private async ValueTask WriteEndMarkerAsync( + uint posState, + CancellationToken cancellationToken = default + ) + { + if (!_writeEndMark) + { + return; + } + + await _isMatch[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState] + .EncodeAsync(_rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + await _isRep[_state._index] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + _state.UpdateMatch(); + var len = Base.K_MATCH_MIN_LEN; + await _lenEncoder + .EncodeAsync(_rangeEncoder, len - Base.K_MATCH_MIN_LEN, posState, cancellationToken) + .ConfigureAwait(false); + uint posSlot = (1 << Base.K_NUM_POS_SLOT_BITS) - 1; + var lenToPosState = Base.GetLenToPosState(len); + await _posSlotEncoder[lenToPosState] + .EncodeAsync(_rangeEncoder, posSlot, cancellationToken) + .ConfigureAwait(false); + var footerBits = 30; + var posReduced = (((uint)1) << footerBits) - 1; + await _rangeEncoder + .EncodeDirectBitsAsync( + posReduced >> Base.K_NUM_ALIGN_BITS, + footerBits - Base.K_NUM_ALIGN_BITS, + cancellationToken + ) + .ConfigureAwait(false); + await _posAlignEncoder + .ReverseEncodeAsync(_rangeEncoder, posReduced & Base.K_ALIGN_MASK, cancellationToken) + .ConfigureAwait(false); + } + private void Flush(uint nowPos) { ReleaseMfStream(); @@ -1262,6 +1405,17 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties _rangeEncoder.FlushStream(); } + private async ValueTask FlushAsync(uint nowPos, CancellationToken cancellationToken = default) + { + ReleaseMfStream(); + await WriteEndMarkerAsync(nowPos & _posStateMask, cancellationToken).ConfigureAwait(false); + for (var i = 0; i < 5; i++) + { + await _rangeEncoder.ShiftLowAsync(cancellationToken).ConfigureAwait(false); + } + await _rangeEncoder.FlushStreamAsync(cancellationToken).ConfigureAwait(false); + } + public void CodeOneBlock(out long inSize, out long outSize, out bool finished) { inSize = 0; @@ -1503,6 +1657,290 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } + public async ValueTask<(long inSize, long outSize, bool finished)> CodeOneBlockAsync( + CancellationToken cancellationToken = default + ) + { + long inSize = 0; + long outSize = 0; + var finished = true; + + if (_inStream != null) + { + _matchFinder.SetStream(_inStream); + _needReleaseMfStream = true; + _inStream = null; + } + + if (_finished) + { + return (inSize, outSize, finished); + } + _finished = true; + + var progressPosValuePrev = _nowPos64; + if (_nowPos64 == 0) + { + if (_trainSize > 0) + { + for ( + ; + _trainSize > 0 && (!_processingMode || !_matchFinder.IsDataStarved); + _trainSize-- + ) + { + _matchFinder.Skip(1); + } + if (_trainSize == 0) + { + _previousByte = _matchFinder.GetIndexByte(-1); + } + } + if (_processingMode && _matchFinder.IsDataStarved) + { + _finished = false; + return (inSize, outSize, finished); + } + if (_matchFinder.GetNumAvailableBytes() == 0) + { + await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false); + return (inSize, outSize, finished); + } + + ReadMatchDistances(out var len, out var numDistancePairs); + var posState = (uint)_nowPos64 & _posStateMask; + await _isMatch[(_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + _state.UpdateChar(); + var curByte = _matchFinder.GetIndexByte((int)(0 - _additionalOffset)); + await _literalEncoder + .GetSubCoder((uint)_nowPos64, _previousByte) + .EncodeAsync(_rangeEncoder, curByte, cancellationToken) + .ConfigureAwait(false); + _previousByte = curByte; + _additionalOffset--; + _nowPos64++; + } + if (_processingMode && _matchFinder.IsDataStarved) + { + _finished = false; + return (inSize, outSize, finished); + } + if (_matchFinder.GetNumAvailableBytes() == 0) + { + await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false); + return (inSize, outSize, finished); + } + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (_processingMode && _matchFinder.IsDataStarved) + { + _finished = false; + return (inSize, outSize, finished); + } + + var len = GetOptimum((uint)_nowPos64, out var pos); + + var posState = (uint)_nowPos64 & _posStateMask; + var complexState = (_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState; + if (len == 1 && pos == 0xFFFFFFFF) + { + await _isMatch[complexState] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + var curByte = _matchFinder.GetIndexByte((int)(0 - _additionalOffset)); + var subCoder = _literalEncoder.GetSubCoder((uint)_nowPos64, _previousByte); + if (!_state.IsCharState()) + { + var matchByte = _matchFinder.GetIndexByte( + (int)(0 - _repDistances[0] - 1 - _additionalOffset) + ); + await subCoder + .EncodeMatchedAsync(_rangeEncoder, matchByte, curByte, cancellationToken) + .ConfigureAwait(false); + } + else + { + await subCoder + .EncodeAsync(_rangeEncoder, curByte, cancellationToken) + .ConfigureAwait(false); + } + _previousByte = curByte; + _state.UpdateChar(); + } + else + { + await _isMatch[complexState] + .EncodeAsync(_rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + if (pos < Base.K_NUM_REP_DISTANCES) + { + await _isRep[_state._index] + .EncodeAsync(_rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + if (pos == 0) + { + await _isRepG0[_state._index] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + await _isRep0Long[complexState] + .EncodeAsync(_rangeEncoder, len == 1 ? 0u : 1u, cancellationToken) + .ConfigureAwait(false); + } + else + { + await _isRepG0[_state._index] + .EncodeAsync(_rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + if (pos == 1) + { + await _isRepG1[_state._index] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + } + else + { + await _isRepG1[_state._index] + .EncodeAsync(_rangeEncoder, 1, cancellationToken) + .ConfigureAwait(false); + await _isRepG2[_state._index] + .EncodeAsync(_rangeEncoder, pos - 2, cancellationToken) + .ConfigureAwait(false); + } + } + if (len == 1) + { + _state.UpdateShortRep(); + } + else + { + await _repMatchLenEncoder + .EncodeAsync( + _rangeEncoder, + len - Base.K_MATCH_MIN_LEN, + posState, + cancellationToken + ) + .ConfigureAwait(false); + _state.UpdateRep(); + } + var distance = _repDistances[pos]; + if (pos != 0) + { + for (var i = pos; i >= 1; i--) + { + _repDistances[i] = _repDistances[i - 1]; + } + _repDistances[0] = distance; + } + } + else + { + await _isRep[_state._index] + .EncodeAsync(_rangeEncoder, 0, cancellationToken) + .ConfigureAwait(false); + _state.UpdateMatch(); + await _lenEncoder + .EncodeAsync( + _rangeEncoder, + len - Base.K_MATCH_MIN_LEN, + posState, + cancellationToken + ) + .ConfigureAwait(false); + pos -= Base.K_NUM_REP_DISTANCES; + var posSlot = GetPosSlot(pos); + var lenToPosState = Base.GetLenToPosState(len); + await _posSlotEncoder[lenToPosState] + .EncodeAsync(_rangeEncoder, posSlot, cancellationToken) + .ConfigureAwait(false); + + if (posSlot >= Base.K_START_POS_MODEL_INDEX) + { + var footerBits = (int)((posSlot >> 1) - 1); + var baseVal = ((2 | (posSlot & 1)) << footerBits); + var posReduced = pos - baseVal; + + if (posSlot < Base.K_END_POS_MODEL_INDEX) + { + await BitTreeEncoder + .ReverseEncodeAsync( + _posEncoders, + baseVal - posSlot - 1, + _rangeEncoder, + footerBits, + posReduced, + cancellationToken + ) + .ConfigureAwait(false); + } + else + { + await _rangeEncoder + .EncodeDirectBitsAsync( + posReduced >> Base.K_NUM_ALIGN_BITS, + footerBits - Base.K_NUM_ALIGN_BITS, + cancellationToken + ) + .ConfigureAwait(false); + await _posAlignEncoder + .ReverseEncodeAsync( + _rangeEncoder, + posReduced & Base.K_ALIGN_MASK, + cancellationToken + ) + .ConfigureAwait(false); + _alignPriceCount++; + } + } + var distance = pos; + for (var i = Base.K_NUM_REP_DISTANCES - 1; i >= 1; i--) + { + _repDistances[i] = _repDistances[i - 1]; + } + _repDistances[0] = distance; + _matchPriceCount++; + } + _previousByte = _matchFinder.GetIndexByte((int)(len - 1 - _additionalOffset)); + } + _additionalOffset -= len; + _nowPos64 += len; + if (_additionalOffset == 0) + { + if (_matchPriceCount >= (1 << 7)) + { + FillDistancesPrices(); + } + if (_alignPriceCount >= Base.K_ALIGN_TABLE_SIZE) + { + FillAlignPrices(); + } + inSize = _nowPos64; + outSize = _rangeEncoder.GetProcessedSizeAdd(); + if (_processingMode && _matchFinder.IsDataStarved) + { + _finished = false; + return (inSize, outSize, finished); + } + if (_matchFinder.GetNumAvailableBytes() == 0) + { + await FlushAsync((uint)_nowPos64, cancellationToken).ConfigureAwait(false); + return (inSize, outSize, finished); + } + + if (_nowPos64 - progressPosValuePrev >= (1 << 12)) + { + _finished = false; + finished = false; + return (inSize, outSize, finished); + } + } + } + } + private void ReleaseMfStream() { if (_matchFinder != null && _needReleaseMfStream) @@ -1599,6 +2037,36 @@ internal class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties } } + public async ValueTask CodeAsync( + Stream inStream, + bool final, + CancellationToken cancellationToken = default + ) + { + _matchFinder.SetStream(inStream); + _processingMode = !final; + try + { + while (true) + { + var (processedInSize, _, finished) = await CodeOneBlockAsync(cancellationToken) + .ConfigureAwait(false); + if (finished) + { + return processedInSize; + } + } + } + finally + { + _matchFinder.ReleaseStream(); + if (final) + { + ReleaseStreams(); + } + } + } + public void Train(Stream trainStream) { if (_nowPos64 > 0) diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 6167c8b2..3adef73d 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -389,7 +389,7 @@ public partial class LzmaStream } #endif - public override Task WriteAsync( + public override async Task WriteAsync( byte[] buffer, int offset, int count, @@ -397,7 +397,63 @@ public partial class LzmaStream ) { cancellationToken.ThrowIfCancellationRequested(); - Write(buffer, offset, count); - return Task.CompletedTask; + if (_encoder != null) + { + _position = await _encoder + .CodeAsync(new MemoryStream(buffer, offset, count), false, cancellationToken) + .ConfigureAwait(false); + } + } + +#if !LEGACY_DOTNET + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_encoder != null) + { + _position = await _encoder + .CodeAsync(new MemoryStream(buffer.ToArray()), false, cancellationToken) + .ConfigureAwait(false); + } + } +#endif + +#if !LEGACY_DOTNET || NETSTANDARD2_1 + public override async ValueTask DisposeAsync() +#else + public async ValueTask DisposeAsync() +#endif + { + if (_isDisposed) + { + return; + } + _isDisposed = true; + + if (_encoder != null) + { + _position = await _encoder.CodeAsync(null, true).ConfigureAwait(false); + } + + if (!_leaveOpen) + { + if (_inputStream is IAsyncDisposable asyncDisposableInputStream) + { + await asyncDisposableInputStream.DisposeAsync().ConfigureAwait(false); + } + else + { + _inputStream?.Dispose(); + } + } + await _outWindow.DisposeAsync().ConfigureAwait(false); + +#if !LEGACY_DOTNET || NETSTANDARD2_1 + await base.DisposeAsync().ConfigureAwait(false); +#endif + GC.SuppressFinalize(this); } } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index f6314379..12f6884e 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -8,7 +8,7 @@ using SharpCompress.IO; namespace SharpCompress.Compressors.LZMA; -public partial class LzmaStream : Stream, IStreamStack +public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable { private readonly Stream? _inputStream; private readonly long _inputSize;