Compare commits

...

9 Commits

Author SHA1 Message Date
Adam Hathcock
895dd02830 another fix 2026-01-31 14:20:01 +00:00
Adam Hathcock
7112dba345 some shrink fixes 2026-01-31 13:56:58 +00:00
Adam Hathcock
0767292bb0 ReduceStream is async 2026-01-31 13:19:10 +00:00
Adam Hathcock
b40e1a002a Merge remote-tracking branch 'origin/adam/data-descriptor-fix' into adam/more-explode-async 2026-01-31 11:18:42 +00:00
Adam Hathcock
c096164486 add shrink stream async 2026-01-31 11:18:16 +00:00
Adam Hathcock
b48e938c98 finish PPMD? 2026-01-30 13:46:30 +00:00
Adam Hathcock
4ed1f89866 more ppmd async 2026-01-30 13:19:24 +00:00
Adam Hathcock
525bcea989 ppmd create 2026-01-30 12:37:21 +00:00
Adam Hathcock
6c3f7c86da lzma works with zip 2026-01-30 12:28:01 +00:00
21 changed files with 1186 additions and 78 deletions

View File

@@ -136,28 +136,55 @@ internal abstract partial class ZipFilePart
}
case ZipCompressionMethod.Shrink:
{
return new ShrinkStream(
stream,
CompressionMode.Decompress,
Header.CompressedSize,
Header.UncompressedSize
);
return await ShrinkStream
.CreateAsync(
stream,
CompressionMode.Decompress,
Header.CompressedSize,
Header.UncompressedSize,
cancellationToken
)
.ConfigureAwait(false);
}
case ZipCompressionMethod.Reduce1:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 1);
return await ReduceStream.CreateAsync(
stream,
Header.CompressedSize,
Header.UncompressedSize,
1,
cancellationToken
);
}
case ZipCompressionMethod.Reduce2:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 2);
return await ReduceStream.CreateAsync(
stream,
Header.CompressedSize,
Header.UncompressedSize,
2,
cancellationToken
);
}
case ZipCompressionMethod.Reduce3:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 3);
return await ReduceStream.CreateAsync(
stream,
Header.CompressedSize,
Header.UncompressedSize,
3,
cancellationToken
);
}
case ZipCompressionMethod.Reduce4:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 4);
return await ReduceStream.CreateAsync(
stream,
Header.CompressedSize,
Header.UncompressedSize,
4,
cancellationToken
);
}
case ZipCompressionMethod.Explode:
{
@@ -201,7 +228,7 @@ internal abstract partial class ZipFilePart
await stream
.ReadFullyAsync(props, 0, propsSize, cancellationToken)
.ConfigureAwait(false);
return LzmaStream.Create(
return await LzmaStream.CreateAsync(
props,
stream,
Header.CompressedSize > 0 ? Header.CompressedSize - 4 - props.Length : -1,
@@ -222,7 +249,9 @@ internal abstract partial class ZipFilePart
{
var props = new byte[2];
await stream.ReadFullyAsync(props, 0, 2, cancellationToken).ConfigureAwait(false);
return new PpmdStream(new PpmdProperties(props), stream, false);
return await PpmdStream
.CreateAsync(new PpmdProperties(props), stream, false, cancellationToken)
.ConfigureAwait(false);
}
case ZipCompressionMethod.WinzipAes:
{

View File

@@ -88,19 +88,39 @@ internal abstract partial class ZipFilePart : FilePart
}
case ZipCompressionMethod.Reduce1:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 1);
return ReduceStream.Create(
stream,
Header.CompressedSize,
Header.UncompressedSize,
1
);
}
case ZipCompressionMethod.Reduce2:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 2);
return ReduceStream.Create(
stream,
Header.CompressedSize,
Header.UncompressedSize,
2
);
}
case ZipCompressionMethod.Reduce3:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 3);
return ReduceStream.Create(
stream,
Header.CompressedSize,
Header.UncompressedSize,
3
);
}
case ZipCompressionMethod.Reduce4:
{
return new ReduceStream(stream, Header.CompressedSize, Header.UncompressedSize, 4);
return ReduceStream.Create(
stream,
Header.CompressedSize,
Header.UncompressedSize,
4
);
}
case ZipCompressionMethod.Explode:
{
@@ -155,7 +175,7 @@ internal abstract partial class ZipFilePart : FilePart
{
Span<byte> props = stackalloc byte[2];
stream.ReadFully(props);
return new PpmdStream(new PpmdProperties(props), stream, false);
return PpmdStream.Create(new PpmdProperties(props), stream, false);
}
case ZipCompressionMethod.WinzipAes:
{

View File

@@ -67,7 +67,14 @@ internal static partial class DecoderRegistry
cancellationToken: cancellationToken
);
case K_PPMD:
return new PpmdStream(new PpmdProperties(info), inStreams.Single(), false);
return await PpmdStream
.CreateAsync(
new PpmdProperties(info),
inStreams.Single(),
false,
cancellationToken
)
.ConfigureAwait(false);
case K_DEFLATE:
return new DeflateStream(inStreams.Single(), CompressionMode.Decompress);
case K_ZSTD:

View File

@@ -8,6 +8,23 @@ namespace SharpCompress.Compressors.LZMA;
public partial class LzmaStream
{
public static ValueTask<LzmaStream> CreateAsync(
byte[] properties,
Stream inputStream,
long inputSize,
long outputSize,
bool leaveOpen = false
) =>
CreateAsync(
properties,
inputStream,
inputSize,
outputSize,
null,
properties.Length < 5,
leaveOpen
);
public static async ValueTask<LzmaStream> CreateAsync(
byte[] properties,
Stream inputStream,

View File

@@ -191,4 +191,15 @@ internal partial class Decoder
await NormalizeAsync(cancellationToken).ConfigureAwait(false);
return symbol;
}
public async ValueTask DecodeAsync(
uint start,
uint size,
CancellationToken cancellationToken = default
)
{
_code -= start * _range;
_range *= size;
await NormalizeAsync(cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -75,7 +75,7 @@ internal static partial class DecoderRegistry
case K_B_ZIP2:
return BZip2Stream.Create(inStreams.Single(), CompressionMode.Decompress, true);
case K_PPMD:
return new PpmdStream(new PpmdProperties(info), inStreams.Single(), false);
return PpmdStream.Create(new PpmdProperties(info), inStreams.Single(), false);
case K_DEFLATE:
return new DeflateStream(inStreams.Single(), CompressionMode.Decompress);
case K_ZSTD:

View File

@@ -3,6 +3,8 @@
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.Rar;
using Decoder = SharpCompress.Compressors.LZMA.RangeCoder.Decoder;
@@ -400,6 +402,77 @@ internal class ModelPpm
return (symbol);
}
public virtual async ValueTask<int> DecodeCharAsync(
CancellationToken cancellationToken = default
)
{
// Debug
//subAlloc.dumpHeap();
if (_minContext.Address <= SubAlloc.PText || _minContext.Address > SubAlloc.HeapEnd)
{
return (-1);
}
if (_minContext.NumStats != 1)
{
if (
_minContext.FreqData.GetStats() <= SubAlloc.PText
|| _minContext.FreqData.GetStats() > SubAlloc.HeapEnd
)
{
return (-1);
}
if (!_minContext.DecodeSymbol1(this))
{
return (-1);
}
}
else
{
_minContext.DecodeBinSymbol(this);
}
Coder.Decode();
while (FoundState.Address == 0)
{
await Coder.AriDecNormalizeAsync(cancellationToken).ConfigureAwait(false);
do
{
_orderFall++;
_minContext.Address = _minContext.GetSuffix(); // =MinContext->Suffix;
if (_minContext.Address <= SubAlloc.PText || _minContext.Address > SubAlloc.HeapEnd)
{
return (-1);
}
} while (_minContext.NumStats == _numMasked);
if (!_minContext.DecodeSymbol2(this))
{
return (-1);
}
Coder.Decode();
}
var symbol = FoundState.Symbol;
if ((_orderFall == 0) && FoundState.GetSuccessor() > SubAlloc.PText)
{
// MinContext=MaxContext=FoundState->Successor;
var addr = FoundState.GetSuccessor();
_minContext.Address = addr;
_maxContext.Address = addr;
}
else
{
UpdateModel();
//this.foundState.Address=foundState.Address);//TODO just 4 debugging
if (_escCount == 0)
{
ClearMask();
}
}
await Coder.AriDecNormalizeAsync(cancellationToken).ConfigureAwait(false); // ARI_DEC_NORMALIZE(Coder.code,Coder.low,Coder.range,Coder.UnpackRead);
return (symbol);
}
public virtual See2Context[][] GetSee2Cont() => _see2Cont;
public virtual void IncEscCount(int dEscCount) => EscCount += dEscCount;
@@ -787,6 +860,43 @@ internal class ModelPpm
return (_minContext.Address != 0);
}
internal async ValueTask<bool> DecodeInitAsync(
Stream stream,
int maxOrder,
int maxMemory,
CancellationToken cancellationToken = default
)
{
if (stream != null)
{
Coder = new RangeCoder();
await Coder.InitAsync(stream, cancellationToken).ConfigureAwait(false);
}
if (maxOrder == 1)
{
SubAlloc.StopSubAllocator();
return (false);
}
SubAlloc.StartSubAllocator(maxMemory);
_minContext = new PpmContext(Heap);
//medContext = new PPMContext(Heap);
_maxContext = new PpmContext(Heap);
FoundState = new State(Heap);
_dummySee2Cont = new See2Context();
for (var i = 0; i < 25; i++)
{
for (var j = 0; j < 16; j++)
{
_see2Cont[i][j] = new See2Context();
}
}
StartModelRare(maxOrder);
return (_minContext.Address != 0);
}
internal void NextContext()
{
var addr = FoundState.GetSuccessor();
@@ -955,4 +1065,179 @@ internal class ModelPpm
} while (i != 0);
}
}
public async ValueTask<int> DecodeCharAsync(
Decoder decoder,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
if (_minContext.NumStats != 1)
{
var s = _tempState1.Initialize(Heap);
s.Address = _minContext.FreqData.GetStats();
int i;
int count,
hiCnt;
if (
(count = (int)decoder.GetThreshold((uint)_minContext.FreqData.SummFreq))
< (hiCnt = s.Freq)
)
{
byte symbol;
await decoder.DecodeAsync(0, (uint)s.Freq, cancellationToken).ConfigureAwait(false);
symbol = (byte)s.Symbol;
_minContext.update1_0(this, s.Address);
NextContext();
return symbol;
}
_prevSuccess = 0;
i = _minContext.NumStats - 1;
do
{
s.IncrementAddress();
if ((hiCnt += s.Freq) > count)
{
byte symbol;
await decoder
.DecodeAsync((uint)(hiCnt - s.Freq), (uint)s.Freq, cancellationToken)
.ConfigureAwait(false);
symbol = (byte)s.Symbol;
_minContext.Update1(this, s.Address);
NextContext();
return symbol;
}
} while (--i > 0);
if (count >= _minContext.FreqData.SummFreq)
{
return -2;
}
_hiBitsFlag = _hb2Flag[FoundState.Symbol];
await decoder
.DecodeAsync(
(uint)hiCnt,
(uint)(_minContext.FreqData.SummFreq - hiCnt),
cancellationToken
)
.ConfigureAwait(false);
for (i = 0; i < 256; i++)
{
_charMask[i] = -1;
}
_charMask[s.Symbol] = 0;
i = _minContext.NumStats - 1;
do
{
s.DecrementAddress();
_charMask[s.Symbol] = 0;
} while (--i > 0);
}
else
{
var rs = _tempState1.Initialize(Heap);
rs.Address = _minContext.GetOneState().Address;
_hiBitsFlag = GetHb2Flag()[FoundState.Symbol];
var off1 = rs.Freq - 1;
var off2 = _minContext.GetArrayIndex(this, rs);
var bs = _binSumm[off1][off2];
if (
await decoder.DecodeBitAsync((uint)bs, 14, cancellationToken).ConfigureAwait(false)
== 0
)
{
byte symbol;
_binSumm[off1][off2] =
(bs + INTERVAL - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF;
FoundState.Address = rs.Address;
symbol = (byte)rs.Symbol;
rs.IncrementFreq((rs.Freq < 128) ? 1 : 0);
_prevSuccess = 1;
IncRunLength(1);
NextContext();
return symbol;
}
bs = (bs - _minContext.GetMean(bs, PERIOD_BITS, 2)) & 0xFFFF;
_binSumm[off1][off2] = bs;
_initEsc = PpmContext.EXP_ESCAPE[Utility.URShift(bs, 10)];
int i;
for (i = 0; i < 256; i++)
{
_charMask[i] = -1;
}
_charMask[rs.Symbol] = 0;
_prevSuccess = 0;
}
for (; ; )
{
var s = _tempState1.Initialize(Heap);
int i;
int count,
hiCnt;
See2Context see;
int num,
numMasked = _minContext.NumStats;
do
{
_orderFall++;
_minContext.Address = _minContext.GetSuffix();
if (_minContext.Address <= SubAlloc.PText || _minContext.Address > SubAlloc.HeapEnd)
{
return -1;
}
} while (_minContext.NumStats == numMasked);
hiCnt = 0;
s.Address = _minContext.FreqData.GetStats();
i = 0;
num = _minContext.NumStats - numMasked;
do
{
var k = _charMask[s.Symbol];
hiCnt += s.Freq & k;
_minContext._ps[i] = s.Address;
s.IncrementAddress();
i -= k;
} while (i != num);
see = _minContext.MakeEscFreq(this, numMasked, out var freqSum);
freqSum += hiCnt;
count = (int)decoder.GetThreshold((uint)freqSum);
if (count < hiCnt)
{
byte symbol;
var ps = _tempState2.Initialize(Heap);
for (
hiCnt = 0, i = 0, ps.Address = _minContext._ps[i];
(hiCnt += ps.Freq) <= count;
i++, ps.Address = _minContext._ps[i]
)
{
;
}
s.Address = ps.Address;
await decoder
.DecodeAsync((uint)(hiCnt - s.Freq), (uint)s.Freq, cancellationToken)
.ConfigureAwait(false);
see.Update();
symbol = (byte)s.Symbol;
_minContext.Update2(this, s.Address);
UpdateModel();
return symbol;
}
if (count >= freqSum)
{
return -2;
}
await decoder
.DecodeAsync((uint)hiCnt, (uint)(freqSum - hiCnt), cancellationToken)
.ConfigureAwait(false);
see.Summ += freqSum;
do
{
s.Address = _minContext._ps[--i];
_charMask[s.Symbol] = 0;
} while (i != 0);
}
}
}

View File

@@ -2,7 +2,10 @@
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.Rar;
using SharpCompress.IO;
namespace SharpCompress.Compressors.PPMd.H;
@@ -31,6 +34,8 @@ internal class RangeCoder
Init();
}
internal RangeCoder() { }
private void Init()
{
SubRange = new SubRange();
@@ -43,6 +48,22 @@ internal class RangeCoder
}
}
internal async ValueTask InitAsync(Stream stream, CancellationToken cancellationToken = default)
{
SubRange = new SubRange();
_low = _code = 0L;
_range = 0xFFFFffffL;
byte[] buffer = new byte[4];
await stream.ReadFullyAsync(buffer, 0, 4, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < 4; i++)
{
_code = ((_code << 8) | buffer[i]) & UINT_MASK;
}
}
internal int CurrentCount
{
get
@@ -106,6 +127,39 @@ internal class RangeCoder
}
}
private async ValueTask<long> ReadCharAsync(CancellationToken cancellationToken = default)
{
if (_unpackRead != null)
{
return _unpackRead.Char;
}
if (_stream != null)
{
byte[] buffer = new byte[1];
await _stream.ReadFullyAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false);
return buffer[0];
}
return -1;
}
internal async ValueTask AriDecNormalizeAsync(CancellationToken cancellationToken = default)
{
var c2 = false;
while ((_low ^ (_low + _range)) < TOP || (c2 = _range < BOT))
{
if (c2)
{
_range = (-_low & (BOT - 1)) & UINT_MASK;
c2 = false;
}
_code =
((_code << 8) | await ReadCharAsync(cancellationToken).ConfigureAwait(false))
& UINT_MASK;
_range = (_range << 8) & UINT_MASK;
_low = (_low << 8) & UINT_MASK;
}
}
// Debug
public override string ToString()
{

View File

@@ -1,6 +1,9 @@
#region Using
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.IO;
#endregion
@@ -76,6 +79,24 @@ internal class Coder
}
}
public async ValueTask RangeDecoderInitializeAsync(
Stream stream,
CancellationToken cancellationToken = default
)
{
_low = 0;
_code = 0;
_range = uint.MaxValue;
byte[] buffer = new byte[4];
await stream.ReadFullyAsync(buffer, 0, 4, cancellationToken).ConfigureAwait(false);
for (uint index = 0; index < 4; index++)
{
_code = (_code << 8) | buffer[index];
}
}
public void RangeDecoderNormalize(Stream stream)
{
while (
@@ -89,6 +110,24 @@ internal class Coder
}
}
public async ValueTask RangeDecoderNormalizeAsync(
Stream stream,
CancellationToken cancellationToken = default
)
{
while (
(_low ^ (_low + _range)) < RANGE_TOP
|| _range < RANGE_BOTTOM && ((_range = (uint)-_low & (RANGE_BOTTOM - 1)) != 0 || true)
)
{
byte[] buffer = new byte[1];
await stream.ReadFullyAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false);
_code = (_code << 8) | buffer[0];
_range <<= 8;
_low <<= 8;
}
}
public uint RangeGetCurrentCount() => (_code - _low) / (_range /= _scale);
public uint RangeGetCurrentShiftCount(int rangeShift) =>

View File

@@ -2,6 +2,8 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
// This is a port of Dmitry Shkarin's PPMd Variant I Revision 1.
// Ported by Michael Bone (mjbone03@yahoo.com.au).
@@ -267,6 +269,21 @@ internal partial class Model
return _coder;
}
internal async ValueTask<Coder> DecodeStartAsync(
Stream source,
PpmdProperties properties,
CancellationToken cancellationToken = default
)
{
_allocator = properties._allocator;
_coder = new Coder();
await _coder.RangeDecoderInitializeAsync(source, cancellationToken).ConfigureAwait(false);
StartModel(properties.ModelOrder, properties.RestorationMethod);
_minimumContext = _maximumContext;
_numberStatistics = _minimumContext.NumberStatistics;
return _coder;
}
internal int DecodeBlock(Stream source, byte[] buffer, int offset, int count)
{
if (_minimumContext == PpmContext.ZERO)
@@ -330,6 +347,81 @@ internal partial class Model
return total;
}
internal async ValueTask<int> DecodeBlockAsync(
Stream source,
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken = default
)
{
if (_minimumContext == PpmContext.ZERO)
{
return 0;
}
var total = 0;
while (total < count)
{
cancellationToken.ThrowIfCancellationRequested();
if (_numberStatistics != 0)
{
DecodeSymbol1(_minimumContext);
}
else
{
DecodeBinarySymbol(_minimumContext);
}
_coder.RangeRemoveSubrange();
while (_foundState == PpmState.ZERO)
{
await _coder
.RangeDecoderNormalizeAsync(source, cancellationToken)
.ConfigureAwait(false);
do
{
_orderFall++;
_minimumContext = _minimumContext.Suffix;
if (_minimumContext == PpmContext.ZERO)
{
goto StopDecoding;
}
} while (_minimumContext.NumberStatistics == _numberMasked);
DecodeSymbol2(_minimumContext);
_coder.RangeRemoveSubrange();
}
buffer[offset] = _foundState.Symbol;
offset++;
total++;
if (_orderFall == 0 && (Pointer)_foundState.Successor >= _allocator._baseUnit)
{
_maximumContext = _foundState.Successor;
}
else
{
UpdateModel(_minimumContext);
if (_escapeCount == 0)
{
ClearMask();
}
}
_minimumContext = _maximumContext;
_numberStatistics = _minimumContext.NumberStatistics;
await _coder
.RangeDecoderNormalizeAsync(source, cancellationToken)
.ConfigureAwait(false);
}
StopDecoding:
return total;
}
#endregion
#region Private Methods

View File

@@ -2,6 +2,8 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Compressors.LZMA.RangeCoder;
using SharpCompress.Compressors.PPMd.H;
using SharpCompress.Compressors.PPMd.I1;
@@ -34,13 +36,13 @@ public class PpmdStream : Stream, IStreamStack
private readonly PpmdProperties _properties;
private readonly Stream _stream;
private readonly bool _compress;
private readonly Model _model;
private readonly ModelPpm _modelH;
private readonly Decoder _decoder;
private Model _model;
private ModelPpm _modelH;
private Decoder _decoder;
private long _position;
private bool _isDisposed;
public PpmdStream(PpmdProperties properties, Stream stream, bool compress)
private PpmdStream(PpmdProperties properties, Stream stream, bool compress)
{
_properties = properties;
_stream = stream;
@@ -50,40 +52,142 @@ public class PpmdStream : Stream, IStreamStack
this.DebugConstruct(typeof(PpmdStream));
#endif
if (properties.Version == PpmdVersion.I1)
InitializeSync(stream, compress);
}
private PpmdStream(
PpmdProperties properties,
Stream stream,
bool compress,
bool skipInitialization
)
{
_properties = properties;
_stream = stream;
_compress = compress;
#if DEBUG_STREAMS
this.DebugConstruct(typeof(PpmdStream));
#endif
// Skip initialization - used by CreateAsync
}
private void InitializeSync(Stream stream, bool compress)
{
if (_properties.Version == PpmdVersion.I1)
{
_model = new Model();
if (compress)
{
_model.EncodeStart(properties);
_model.EncodeStart(_properties);
}
else
{
_model.DecodeStart(stream, properties);
_model.DecodeStart(stream, _properties);
}
}
if (properties.Version == PpmdVersion.H)
if (_properties.Version == PpmdVersion.H)
{
_modelH = new ModelPpm();
if (compress)
{
throw new NotImplementedException();
}
_modelH.DecodeInit(stream, properties.ModelOrder, properties.AllocatorSize);
_modelH.DecodeInit(stream, _properties.ModelOrder, _properties.AllocatorSize);
}
if (properties.Version == PpmdVersion.H7Z)
if (_properties.Version == PpmdVersion.H7Z)
{
_modelH = new ModelPpm();
if (compress)
{
throw new NotImplementedException();
}
_modelH.DecodeInit(null, properties.ModelOrder, properties.AllocatorSize);
_modelH.DecodeInit(null, _properties.ModelOrder, _properties.AllocatorSize);
_decoder = new Decoder();
_decoder.Init(stream);
}
}
public static PpmdStream Create(PpmdProperties properties, Stream stream, bool compress) =>
new PpmdStream(properties, stream, compress);
public static async ValueTask<PpmdStream> CreateAsync(
PpmdProperties properties,
Stream stream,
bool compress,
CancellationToken cancellationToken = default
)
{
if (stream is null)
{
throw new ArgumentNullException(nameof(stream));
}
if (properties.Version == PpmdVersion.H && compress)
{
throw new NotImplementedException("PPMd H version compression not supported");
}
if (properties.Version == PpmdVersion.H7Z && compress)
{
throw new NotImplementedException("PPMd H7Z version compression not supported");
}
var instance = new PpmdStream(properties, stream, compress, skipInitialization: true);
try
{
if (properties.Version == PpmdVersion.I1)
{
instance._model = new Model();
if (compress)
{
instance._model.EncodeStart(properties);
}
else
{
await instance
._model.DecodeStartAsync(stream, properties, cancellationToken)
.ConfigureAwait(false);
}
}
else if (properties.Version == PpmdVersion.H)
{
instance._modelH = new ModelPpm();
await instance
._modelH.DecodeInitAsync(
stream,
properties.ModelOrder,
properties.AllocatorSize,
cancellationToken
)
.ConfigureAwait(false);
}
else if (properties.Version == PpmdVersion.H7Z)
{
instance._modelH = new ModelPpm();
await instance
._modelH.DecodeInitAsync(
null,
properties.ModelOrder,
properties.AllocatorSize,
cancellationToken
)
.ConfigureAwait(false);
instance._decoder = new Decoder();
await instance._decoder.InitAsync(stream, cancellationToken).ConfigureAwait(false);
}
return instance;
}
catch
{
instance.Dispose();
throw;
}
}
public override bool CanRead => !_compress;
public override bool CanSeek => false;
@@ -157,6 +261,118 @@ public class PpmdStream : Stream, IStreamStack
public override void SetLength(long value) => throw new NotSupportedException();
public override async Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
if (_compress)
{
return 0;
}
cancellationToken.ThrowIfCancellationRequested();
var size = 0;
if (_properties.Version == PpmdVersion.I1)
{
size = await _model
.DecodeBlockAsync(_stream, buffer, offset, count, cancellationToken)
.ConfigureAwait(false);
}
if (_properties.Version == PpmdVersion.H)
{
int c;
while (
size < count
&& (c = await _modelH.DecodeCharAsync(cancellationToken).ConfigureAwait(false)) >= 0
)
{
buffer[offset++] = (byte)c;
size++;
}
}
if (_properties.Version == PpmdVersion.H7Z)
{
int c;
while (
size < count
&& (
c = await _modelH
.DecodeCharAsync(_decoder, cancellationToken)
.ConfigureAwait(false)
) >= 0
)
{
buffer[offset++] = (byte)c;
size++;
}
}
_position += size;
return size;
}
#if !LEGACY_DOTNET
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
if (_compress)
{
return 0;
}
cancellationToken.ThrowIfCancellationRequested();
var size = 0;
var offset = 0;
var count = buffer.Length;
if (_properties.Version == PpmdVersion.I1)
{
// Need to use a temporary buffer since DecodeBlockAsync works with byte[]
var tempBuffer = new byte[count];
size = await _model
.DecodeBlockAsync(_stream, tempBuffer, 0, count, cancellationToken)
.ConfigureAwait(false);
tempBuffer.AsMemory(0, size).CopyTo(buffer);
}
if (_properties.Version == PpmdVersion.H)
{
int c;
while (
size < count
&& (c = await _modelH.DecodeCharAsync(cancellationToken).ConfigureAwait(false)) >= 0
)
{
buffer.Span[offset++] = (byte)c;
size++;
}
}
if (_properties.Version == PpmdVersion.H7Z)
{
int c;
while (
size < count
&& (
c = await _modelH
.DecodeCharAsync(_decoder, cancellationToken)
.ConfigureAwait(false)
) >= 0
)
{
buffer.Span[offset++] = (byte)c;
size++;
}
}
_position += size;
return size;
}
#endif
public override void Write(byte[] buffer, int offset, int count)
{
if (_compress)

View File

@@ -0,0 +1,206 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Reduce;
public partial class ReduceStream
{
public static async ValueTask<ReduceStream> CreateAsync(
Stream inStr,
long compsize,
long unCompSize,
int factor,
CancellationToken cancellationToken = default
)
{
var stream = new ReduceStream(inStr, compsize, unCompSize, factor);
await stream.LoadNextByteTableAsync(cancellationToken).ConfigureAwait(false);
return stream;
}
private async Task<int> NEXTBYTEAsync(CancellationToken cancellationToken)
{
if (inByteCount == compressedSize)
{
return EOF;
}
byte[] buffer = new byte[1];
int bytesRead = await inStream
.ReadAsync(buffer, 0, 1, cancellationToken)
.ConfigureAwait(false);
if (bytesRead == 0)
{
return EOF;
}
inByteCount++;
return buffer[0];
}
private async Task<byte> READBITSAsync(int nbits, CancellationToken cancellationToken)
{
if (nbits > bitBufferCount)
{
int temp;
while (bitBufferCount <= 8 * (int)(4 - 1))
{
temp = await NEXTBYTEAsync(cancellationToken).ConfigureAwait(false);
if (temp == EOF)
{
break;
}
bitBuffer |= (ulong)temp << bitBufferCount;
bitBufferCount += 8;
}
}
byte zdest = (byte)(bitBuffer & (ulong)mask_bits[nbits]);
bitBuffer >>= nbits;
bitBufferCount -= nbits;
return zdest;
}
private async Task LoadNextByteTableAsync(CancellationToken cancellationToken)
{
nextByteTable = new byte[256][];
for (int x = 255; x >= 0; x--)
{
byte Slen = await READBITSAsync(6, cancellationToken).ConfigureAwait(false);
nextByteTable[x] = new byte[Slen];
for (int i = 0; i < Slen; i++)
{
nextByteTable[x][i] = await READBITSAsync(8, cancellationToken)
.ConfigureAwait(false);
}
}
}
private async Task<byte> GetNextByteAsync(CancellationToken cancellationToken)
{
if (nextByteTable[outByte].Length == 0)
{
outByte = await READBITSAsync(8, cancellationToken).ConfigureAwait(false);
return outByte;
}
byte nextBit = await READBITSAsync(1, cancellationToken).ConfigureAwait(false);
if (nextBit == 1)
{
outByte = await READBITSAsync(8, cancellationToken).ConfigureAwait(false);
return outByte;
}
byte nextByteIndex = await READBITSAsync(
bitCountTable[nextByteTable[outByte].Length],
cancellationToken
)
.ConfigureAwait(false);
outByte = nextByteTable[outByte][nextByteIndex];
return outByte;
}
public override async Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
int countIndex = 0;
while (countIndex < count && outBytesCount < unCompressedSize)
{
if (length == 0)
{
byte nextByte = await GetNextByteAsync(cancellationToken).ConfigureAwait(false);
if (nextByte != RunLengthCode)
{
buffer[offset + (countIndex++)] = nextByte;
windowsBuffer[windowIndex++] = nextByte;
outBytesCount++;
if (windowIndex == WSIZE)
{
windowIndex = 0;
}
continue;
}
nextByte = await GetNextByteAsync(cancellationToken).ConfigureAwait(false);
if (nextByte == 0)
{
buffer[offset + (countIndex++)] = RunLengthCode;
windowsBuffer[windowIndex++] = RunLengthCode;
outBytesCount++;
if (windowIndex == WSIZE)
{
windowIndex = 0;
}
continue;
}
int lengthDistanceByte = nextByte;
length = lengthDistanceByte & lengthMask;
if (length == lengthMask)
{
length += await GetNextByteAsync(cancellationToken).ConfigureAwait(false);
}
length += 3;
int distanceHighByte = (lengthDistanceByte << factor) & distanceMask;
distance =
windowIndex
- (
distanceHighByte
+ await GetNextByteAsync(cancellationToken).ConfigureAwait(false)
+ 1
);
distance &= WSIZE - 1;
}
while (length != 0 && countIndex < count)
{
byte nextByte = windowsBuffer[distance++];
buffer[offset + (countIndex++)] = nextByte;
windowsBuffer[windowIndex++] = nextByte;
outBytesCount++;
if (distance == WSIZE)
{
distance = 0;
}
if (windowIndex == WSIZE)
{
windowIndex = 0;
}
length--;
}
}
return countIndex;
}
#if !LEGACY_DOTNET
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
if (buffer.IsEmpty || outBytesCount >= unCompressedSize)
{
return 0;
}
byte[] arrayBuffer = new byte[buffer.Length];
int result = await ReadAsync(arrayBuffer, 0, arrayBuffer.Length, cancellationToken)
.ConfigureAwait(false);
arrayBuffer.AsMemory(0, result).CopyTo(buffer);
return result;
}
#endif
}

View File

@@ -4,7 +4,7 @@ using SharpCompress.IO;
namespace SharpCompress.Compressors.Reduce;
public class ReduceStream : Stream, IStreamStack
public partial class ReduceStream : Stream, IStreamStack
{
#if DEBUG_STREAMS
long IStreamStack.InstanceId { get; set; }
@@ -44,7 +44,7 @@ public class ReduceStream : Stream, IStreamStack
private int length;
private int distance;
public ReduceStream(Stream inStr, long compsize, long unCompSize, int factor)
private ReduceStream(Stream inStr, long compsize, long unCompSize, int factor)
{
inStream = inStr;
compressedSize = compsize;
@@ -69,7 +69,13 @@ public class ReduceStream : Stream, IStreamStack
outByte = 0;
LoadBitLengthTable();
LoadNextByteTable();
}
public static ReduceStream Create(Stream inStr, long compsize, long unCompSize, int factor)
{
var stream = new ReduceStream(inStr, compsize, unCompSize, factor);
stream.LoadNextByteTable();
return stream;
}
protected override void Dispose(bool disposing)

View File

@@ -2,6 +2,7 @@ namespace SharpCompress.Compressors.Shrink
{
internal class BitStream
{
private const int EOF = 1234;
private byte[] _src;
private int _srcLen;
private int _byteIdx;
@@ -43,7 +44,7 @@ namespace SharpCompress.Compressors.Shrink
{
if (_byteIdx >= _srcLen)
{
return 0;
return EOF;
}
return _src[_byteIdx++];

View File

@@ -275,7 +275,7 @@ namespace SharpCompress.Compressors.Shrink
if (dstPos == dstCap)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
dstUsed = dstPos;
return UnshrnkStatus.Full;
}
@@ -297,7 +297,7 @@ namespace SharpCompress.Compressors.Shrink
if (dstPos == dstCap)
{
srcUsed = stream.BytesRead;
dstUsed = 0;
dstUsed = dstPos;
return UnshrnkStatus.Full;
}

View File

@@ -0,0 +1,133 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Shrink;
internal partial class ShrinkStream : Stream, IStreamStack
{
internal static async ValueTask<ShrinkStream> CreateAsync(
Stream stream,
CompressionMode compressionMode,
long compressedSize,
long uncompressedSize,
CancellationToken cancellationToken = default
)
{
var shrinkStream = new ShrinkStream(
stream,
compressionMode,
compressedSize,
uncompressedSize
);
await shrinkStream.DecompressAsync(cancellationToken).ConfigureAwait(false);
return shrinkStream;
}
private async Task DecompressAsync(CancellationToken cancellationToken)
{
if (_decompressed)
{
return;
}
// Read all compressed data asynchronously
var src = new byte[_compressedSize];
int bytesRead = 0;
int totalBytesRead = 0;
while (totalBytesRead < (int)_compressedSize)
{
bytesRead = await inStream
.ReadAsync(
src,
totalBytesRead,
(int)_compressedSize - totalBytesRead,
cancellationToken
)
.ConfigureAwait(false);
if (bytesRead == 0)
{
throw new EndOfStreamException(
"Unexpected end of stream while reading compressed data"
);
}
totalBytesRead += bytesRead;
}
// Decompress synchronously (CPU-bound operation)
var srcUsed = 0;
var dstUsed = 0;
HwUnshrink.Unshrink(
src,
(int)_compressedSize,
out srcUsed,
_byteOut,
(int)_uncompressedSize,
out dstUsed
);
_outBytesCount = dstUsed;
_decompressed = true;
}
public override async Task<int> ReadAsync(
byte[] buffer,
int offset,
int count,
CancellationToken cancellationToken
)
{
cancellationToken.ThrowIfCancellationRequested();
if (!_decompressed)
{
await DecompressAsync(cancellationToken).ConfigureAwait(false);
}
// Copy from decompressed buffer
long remaining = _outBytesCount - _position;
if (remaining <= 0)
{
return 0;
}
int toCopy = (int)Math.Min(count, remaining);
Buffer.BlockCopy(_byteOut, (int)_position, buffer, offset, toCopy);
_position += toCopy;
return toCopy;
}
#if !LEGACY_DOTNET
public override async ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
if (!_decompressed)
{
await DecompressAsync(cancellationToken).ConfigureAwait(false);
}
if (buffer.IsEmpty)
{
return 0;
}
long remaining = _outBytesCount - _position;
if (remaining <= 0)
{
return 0;
}
int toCopy = (int)Math.Min(buffer.Length, remaining);
_byteOut.AsMemory((int)_position, toCopy).CopyTo(buffer);
_position += toCopy;
return toCopy;
}
#endif
}

View File

@@ -1,10 +1,11 @@
using System;
using System.IO;
using SharpCompress;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Shrink;
internal class ShrinkStream : Stream, IStreamStack
internal partial class ShrinkStream : Stream, IStreamStack
{
#if DEBUG_STREAMS
long IStreamStack.InstanceId { get; set; }
@@ -33,6 +34,8 @@ internal class ShrinkStream : Stream, IStreamStack
private long _uncompressedSize;
private byte[] _byteOut;
private long _outBytesCount;
private bool _decompressed;
private long _position;
public ShrinkStream(
Stream stream,
@@ -72,7 +75,7 @@ internal class ShrinkStream : Stream, IStreamStack
public override long Position
{
get => _outBytesCount;
get => _position;
set => throw new NotImplementedException();
}
@@ -80,32 +83,36 @@ internal class ShrinkStream : Stream, IStreamStack
public override int Read(byte[] buffer, int offset, int count)
{
if (inStream.Position == (long)_compressedSize)
if (!_decompressed)
{
var src = new byte[_compressedSize];
inStream.ReadExact(src, 0, (int)_compressedSize);
var srcUsed = 0;
var dstUsed = 0;
HwUnshrink.Unshrink(
src,
(int)_compressedSize,
out srcUsed,
_byteOut,
(int)_uncompressedSize,
out dstUsed
);
_outBytesCount = dstUsed;
_decompressed = true;
_position = 0;
}
long remaining = _outBytesCount - _position;
if (remaining <= 0)
{
return 0;
}
var src = new byte[_compressedSize];
inStream.Read(src, offset, (int)_compressedSize);
var srcUsed = 0;
var dstUsed = 0;
HwUnshrink.Unshrink(
src,
(int)_compressedSize,
out srcUsed,
_byteOut,
(int)_uncompressedSize,
out dstUsed
);
_outBytesCount = _byteOut.Length;
for (var index = 0; index < _outBytesCount; ++index)
{
buffer[offset + index] = _byteOut[index];
}
var tmp = _outBytesCount;
_outBytesCount = 0;
return (int)tmp;
int toCopy = (int)Math.Min(count, remaining);
Buffer.BlockCopy(_byteOut, (int)_position, buffer, offset, toCopy);
_position += toCopy;
return toCopy;
}
public override long Seek(long offset, SeekOrigin origin) =>

View File

@@ -80,19 +80,12 @@ public class ZipFactory
var startPosition = stream.CanSeek ? stream.Position : -1;
// probe for single volume zip
if (stream is not SharpCompressStream) // wrap to provide buffer bef
{
stream = new SharpCompressStream(stream, bufferSize: Constants.BufferSize);
}
if (await ZipArchive.IsZipFileAsync(stream, password, cancellationToken))
{
return true;
}
// probe for a multipart zip
if (!stream.CanSeek)
{
return false;

View File

@@ -10,7 +10,6 @@ internal partial class RewindableStream : Stream
private bool isRewound;
private bool isDisposed;
private long streamPosition;
private bool _hasStoppedRecording;
public RewindableStream(Stream stream) => this.stream = stream;
@@ -47,7 +46,6 @@ internal partial class RewindableStream : Stream
"StopRecording can only be called when recording is active."
);
}
_hasStoppedRecording = true;
isRewound = true;
IsRecording = false;
bufferStream.Position = 0;
@@ -74,12 +72,6 @@ internal partial class RewindableStream : Stream
"StartRecording can only be called when not already recording."
);
}
if (_hasStoppedRecording)
{
throw new InvalidOperationException(
"StartRecording cannot be called after StopRecording has been called."
);
}
//if (isRewound && bufferStream.Position != 0)
// throw new System.NotImplementedException();
if (bufferStream.Position != 0)

View File

@@ -455,7 +455,7 @@ public partial class ZipWriter : AbstractWriter
case ZipCompressionMethod.PPMd:
{
counting.Write(writer.PpmdProperties.Properties, 0, 2);
return new PpmdStream(writer.PpmdProperties, counting, true);
return PpmdStream.Create(writer.PpmdProperties, counting, true);
}
case ZipCompressionMethod.ZStandard:
{

View File

@@ -136,7 +136,7 @@ public class DisposalTests
// PpmdStream seems to not dispose inner stream based on code analysis
// It takes PpmdProperties which we need to mock or create.
var props = new PpmdProperties();
VerifyNeverDispose(stream => new PpmdStream(props, stream, false));
VerifyNeverDispose(stream => PpmdStream.Create(props, stream, false));
}
[Fact]
@@ -174,7 +174,7 @@ public class DisposalTests
public void ReduceStream_Disposal()
{
// ReduceStream does not dispose inner stream
VerifyNeverDispose(stream => new ReduceStream(stream, 0, 0, 1));
VerifyNeverDispose(stream => ReduceStream.Create(stream, 0, 0, 1));
}
[Fact]