starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-22 15:04:43 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

BZip2: add opt-in tolerateTruncatedStream to decode a footerless/partial stream

Browse Source 
Add a `tolerateTruncatedStream` option (default false) to `BZip2Stream.Create`/`CreateAsync`,
threaded through to `CBZip2InputStream`. When enabled, the decoder accepts a stream that has no
trailing footer - e.g. a truncated stream, or a sub-range of blocks extracted for random access:

- An end-of-input reached while reading a block/footer header (a true block boundary, tracked by
  `expectingBlockStart`) is treated as a normal end of stream instead of throwing
  `ArchiveOperationException("BZip2 compressed file ends unexpectedly")`. EOF in the middle of a
  block, the block CRC, or the Huffman tables still throws.
- The whole-stream combined CRC in the footer is not verified, since a partial decode's running
  combined CRC won't match the stored whole-stream value. Per-block CRCs are still enforced.

Default behaviour is unchanged (the flag defaults to false) and the change is applied symmetrically
to the sync and async read paths.

Tests (BZip2StreamTests):
- a footerless header-only stream decodes to empty with the flag and throws without it;
- a real block followed by end-of-input at the next block boundary decodes with the flag and throws
  without it;
- a corrupted whole-stream combined CRC is tolerated with the flag and fatal without it;
- a complete, well-formed stream still round-trips with the flag set.
All existing BZip2 tests continue to pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Fidel
2026-06-23 12:36:15 +10:00
parent 37a22a8697
commit de8ee30b17
6 changed files with 246 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/SharpCompress/Compressors/BZip2/BZip2Stream.Async.cs
View File

@@ -25,12 +25,22 @@ public sealed partial class BZip2Stream : IAsyncDisposable
/// <param name="stream">The stream to read from</param>
/// <param name="compressionMode">Compression Mode</param>
/// <param name="decompressConcatenated">Decompress Concatenated</param>
/// <param name="leaveOpen">Leave the underlying stream open when this stream is disposed</param>
/// <param name="tolerateTruncatedStream">
/// Decompression only. When true, an end-of-stream reached at a bzip2 block boundary is treated as a
/// normal end of stream rather than throwing. This allows decoding a truncated or partial stream - for
/// example a sub-range of blocks extracted for random access - that has no trailing stream footer. EOF
/// in the middle of a block is still reported as an error. Because a partial decode's running combined
/// CRC won't match the whole-stream value stored in the footer, that whole-stream CRC is not verified
/// in this mode (per-block CRCs are still checked).
/// </param>
/// <param name="cancellationToken">Cancellation Token</param>
public static async ValueTask<BZip2Stream> CreateAsync(
Stream stream,
CompressionMode compressionMode,
bool decompressConcatenated,
bool leaveOpen = false,
bool tolerateTruncatedStream = false,
CancellationToken cancellationToken = default
)
{
@@ -43,7 +53,13 @@ public sealed partial class BZip2Stream : IAsyncDisposable
else
{
bZip2Stream.stream = await CBZip2InputStream
.CreateAsync(stream, decompressConcatenated, leaveOpen, cancellationToken)
.CreateAsync(
stream,
decompressConcatenated,
leaveOpen,
tolerateTruncatedStream,
cancellationToken
)
.ConfigureAwait(false);
}

15
src/SharpCompress/Compressors/BZip2/BZip2Stream.cs
View File

@@ -20,11 +20,21 @@ public sealed partial class BZip2Stream : Stream, IFinishable
/// <param name="stream">The stream to read from</param>
/// <param name="compressionMode">Compression Mode</param>
/// <param name="decompressConcatenated">Decompress Concatenated</param>
/// <param name="leaveOpen">Leave the underlying stream open when this stream is disposed</param>
/// <param name="tolerateTruncatedStream">
/// Decompression only. When true, an end-of-stream reached at a bzip2 block boundary is treated as a
/// normal end of stream rather than throwing. This allows decoding a truncated or partial stream - for
/// example a sub-range of blocks extracted for random access - that has no trailing stream footer. EOF
/// in the middle of a block is still reported as an error. Because a partial decode's running combined
/// CRC won't match the whole-stream value stored in the footer, that whole-stream CRC is not verified
/// in this mode (per-block CRCs are still checked).
/// </param>
public static BZip2Stream Create(
Stream stream,
CompressionMode compressionMode,
bool decompressConcatenated,
bool leaveOpen = false
bool leaveOpen = false,
bool tolerateTruncatedStream = false
)
{
var bZip2Stream = new BZip2Stream();
@@ -38,7 +48,8 @@ public sealed partial class BZip2Stream : Stream, IFinishable
bZip2Stream.stream = CBZip2InputStream.Create(
stream,
decompressConcatenated,
leaveOpen
leaveOpen,
tolerateTruncatedStream
);
}

42
src/SharpCompress/Compressors/BZip2/CBZip2InputStream.Async.cs
View File

@@ -99,12 +99,25 @@ internal partial class CBZip2InputStream
while (true)
{
// A clean EOF is only acceptable here, at the start of a block/footer header.
expectingBlockStart = tolerateTruncatedStream;
magic1 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
magic2 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
magic3 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
magic4 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
magic5 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
magic6 = await BsGetUCharAsync(cancellationToken).ConfigureAwait(false);
expectingBlockStart = false;
if (hitEof)
{
// tolerateTruncatedStream: the input ended at a block boundary (no stream footer). Treat
// it as the end of the stream rather than throwing.
BsFinishedWithStream();
streamEnd = true;
return;
}
if (
magic1 != 0x17
|| magic2 != 0x72
@@ -158,7 +171,9 @@ internal partial class CBZip2InputStream
private async ValueTask<bool> CompleteAsync(CancellationToken cancellationToken)
{
storedCombinedCRC = await BsGetInt32Async(cancellationToken).ConfigureAwait(false);
if (storedCombinedCRC != computedCombinedCRC)
// See Complete() in CBZip2InputStream.cs: the whole-stream combined CRC is not verified in
// tolerateTruncatedStream mode (a partial decode won't match); per-block CRCs still are.
if (!tolerateTruncatedStream && storedCombinedCRC != computedCombinedCRC)
{
CrcError();
}
@@ -877,7 +892,11 @@ internal partial class CBZip2InputStream
{
if (bsStream is null)
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
int zzi;
int thech = '\0';
@@ -889,7 +908,11 @@ internal partial class CBZip2InputStream
}
catch (IOException)
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
finally
{
@@ -897,7 +920,11 @@ internal partial class CBZip2InputStream
}
if (thech == '\uffff')
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
zzi = thech;
bsBuff = (bsBuff << 8) | (zzi & 0xff);
@@ -924,10 +951,15 @@ internal partial class CBZip2InputStream
Stream zStream,
bool decompressConcatenated,
bool leaveOpen = false,
bool tolerateTruncatedStream = false,
CancellationToken cancellationToken = default
)
{
var cbZip2InputStream = new CBZip2InputStream(decompressConcatenated, leaveOpen);
var cbZip2InputStream = new CBZip2InputStream(
decompressConcatenated,
leaveOpen,
tolerateTruncatedStream
);
cbZip2InputStream.ll8 = null;
cbZip2InputStream.tt = null;
cbZip2InputStream.BsSetStream(zStream);

78
src/SharpCompress/Compressors/BZip2/CBZip2InputStream.cs
View File

@@ -58,6 +58,19 @@ internal partial class CBZip2InputStream : Stream
throw new ArchiveOperationException("BZip2 compressed file ends unexpectedly");
}
// Handles the underlying stream running out while BsR needs more bits. In tolerateTruncatedStream
// mode an EOF that lands on a block boundary (expectingBlockStart) ends the stream cleanly; anywhere
// else - or when not tolerating truncation - it is an unexpected truncation and throws.
private void HandleCompressedStreamEof()
{
if (tolerateTruncatedStream && expectingBlockStart)
{
hitEof = true;
return;
}
CompressedStreamEOF();
}
private void MakeMaps()
{
int i;
@@ -151,6 +164,18 @@ internal partial class CBZip2InputStream : Stream
private readonly bool decompressConcatenated;
private readonly bool leaveOpen;
// When true, an end-of-stream reached at a bzip2 block boundary (i.e. while reading a block header)
// is treated as a normal end of stream rather than throwing. Lets a caller decode a truncated or
// partial stream - e.g. a sub-range of blocks extracted for random access - that has no stream footer.
private readonly bool tolerateTruncatedStream;
// true only while reading the 6 bytes of a block/stream-footer header, where a clean EOF is allowed
// (in tolerateTruncatedStream mode). EOF anywhere else is still an unexpected truncation.
private bool expectingBlockStart;
// set by BsR when a tolerated end-of-stream is hit; checked by InitBlock to end the stream.
private bool hitEof;
private int i2,
count,
chPrev,
@@ -163,19 +188,29 @@ internal partial class CBZip2InputStream : Stream
private char z;
private bool isDisposed;
private CBZip2InputStream(bool decompressConcatenated, bool leaveOpen)
private CBZip2InputStream(
bool decompressConcatenated,
bool leaveOpen,
bool tolerateTruncatedStream
)
{
this.decompressConcatenated = decompressConcatenated;
this.leaveOpen = leaveOpen;
this.tolerateTruncatedStream = tolerateTruncatedStream;
}
public static CBZip2InputStream Create(
Stream zStream,
bool decompressConcatenated,
bool leaveOpen
bool leaveOpen,
bool tolerateTruncatedStream = false
)
{
var cbZip2InputStream = new CBZip2InputStream(decompressConcatenated, leaveOpen);
var cbZip2InputStream = new CBZip2InputStream(
decompressConcatenated,
leaveOpen,
tolerateTruncatedStream
);
cbZip2InputStream.ll8 = null;
cbZip2InputStream.tt = null;
cbZip2InputStream.BsSetStream(zStream);
@@ -290,12 +325,25 @@ internal partial class CBZip2InputStream : Stream
while (true)
{
// A clean EOF is only acceptable here, at the start of a block/footer header.
expectingBlockStart = tolerateTruncatedStream;
magic1 = BsGetUChar();
magic2 = BsGetUChar();
magic3 = BsGetUChar();
magic4 = BsGetUChar();
magic5 = BsGetUChar();
magic6 = BsGetUChar();
expectingBlockStart = false;
if (hitEof)
{
// tolerateTruncatedStream: the input ended at a block boundary (no stream footer). Treat
// it as the end of the stream rather than throwing.
BsFinishedWithStream();
streamEnd = true;
return;
}
if (
magic1 != 0x17
|| magic2 != 0x72
@@ -362,7 +410,11 @@ internal partial class CBZip2InputStream : Stream
private bool Complete()
{
storedCombinedCRC = BsGetInt32();
if (storedCombinedCRC != computedCombinedCRC)
// In tolerateTruncatedStream mode the input may be only part of a stream (e.g. a sub-range of
// blocks decoded for random access), so the running combined CRC won't match the stored
// whole-stream value in the footer. Per-block CRCs are still validated; only this whole-stream
// check is skipped.
if (!tolerateTruncatedStream && storedCombinedCRC != computedCombinedCRC)
{
CrcError();
}
@@ -408,7 +460,11 @@ internal partial class CBZip2InputStream : Stream
{
if (bsStream is null)
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
int zzi;
int thech = '\0';
@@ -418,11 +474,19 @@ internal partial class CBZip2InputStream : Stream
}
catch (IOException)
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
if (thech == '\uffff')
{
CompressedStreamEOF();
HandleCompressedStreamEof();
if (hitEof)
{
return 0;
}
}
zzi = thech;
bsBuff = (bsBuff << 8) | (zzi & 0xff);

16
src/SharpCompress/Providers/Default/BZip2CompressionProvider.cs
View File

@@ -30,7 +30,13 @@ public sealed class BZip2CompressionProvider : CompressionProviderBase
{
// BZip2 doesn't use compressionLevel parameter in this implementation
return await BZip2Stream
.CreateAsync(destination, CompressionMode.Compress, false, false, cancellationToken)
.CreateAsync(
destination,
CompressionMode.Compress,
false,
false,
cancellationToken: cancellationToken
)
.ConfigureAwait(false);
}
@@ -45,7 +51,13 @@ public sealed class BZip2CompressionProvider : CompressionProviderBase
)
{
return await BZip2Stream
.CreateAsync(source, CompressionMode.Decompress, false, false, cancellationToken)
.CreateAsync(
source,
CompressionMode.Decompress,
false,
false,
cancellationToken: cancellationToken
)
.ConfigureAwait(false);
}
}

94
tests/SharpCompress.Test/BZip2/BZip2StreamTests.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.IO;
using System.Text;
using SharpCompress.Common;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
using Xunit;
@@ -24,6 +26,98 @@ public class BZip2StreamTests
Assert.Throws<ArchiveOperationException>(() => stream.CopyTo(output));
}
// A stream that ends exactly where a block header is expected (no stream footer) is the shape a caller
// sees when decoding a truncated stream or a sub-range of blocks extracted for random access. "BZh9" is
// a valid header with no blocks and no footer, so the very first block-header read hits end-of-input.
[Fact]
public void BZip2Stream_TolerateTruncatedStream_DecodesFooterlessStreamAsEmpty()
{
var headerOnly = Encoding.ASCII.GetBytes("BZh9");
Assert.Throws<ArchiveOperationException>(() =>
Decompress(headerOnly, tolerateTruncatedStream: false)
);
Assert.Empty(Decompress(headerOnly, tolerateTruncatedStream: true));
}
// A real block followed by end-of-input at the next block boundary: decode a complete stream, then
// append another header that stops before its first block. With tolerance the first stream's data comes
// back and the truncated continuation ends cleanly; without it, the end-of-input throws.
[Fact]
public void BZip2Stream_TolerateTruncatedStream_DecodesStreamTruncatedAtBlockBoundary()
{
const string text = "Some data that bzip2 will put into a single block.";
var truncated = Concat(Compress(text), Encoding.ASCII.GetBytes("BZh9"));
Assert.Throws<ArchiveOperationException>(() =>
Decompress(truncated, tolerateTruncatedStream: false, decompressConcatenated: true)
);
var result = Decompress(
truncated,
tolerateTruncatedStream: true,
decompressConcatenated: true
);
Assert.Equal(text, Encoding.ASCII.GetString(result));
}
// A partial decode's running combined CRC won't match the whole-stream value in the footer, so the
// flag skips that whole-stream check (per-block CRCs are still enforced). Corrupting the stored
// combined CRC is fatal by default but tolerated with the flag.
[Fact]
public void BZip2Stream_TolerateTruncatedStream_SkipsWholeStreamCrc()
{
const string text = "BZip2 combined-CRC validation test data.";
var compressed = Compress(text);
compressed[^5] ^= 1;
Assert.Throws<ArchiveOperationException>(() =>
Decompress(compressed, tolerateTruncatedStream: false)
);
var result = Decompress(compressed, tolerateTruncatedStream: true);
Assert.Equal(text, Encoding.ASCII.GetString(result));
}
// The flag must not change decoding of a normal, well-formed stream.
[Fact]
public void BZip2Stream_TolerateTruncatedStream_StillDecodesCompleteStream()
{
const string text =
"Round trip with tolerateTruncatedStream set on a complete, valid stream.";
var result = Decompress(Compress(text), tolerateTruncatedStream: true);
Assert.Equal(text, Encoding.ASCII.GetString(result));
}
private static byte[] Decompress(
byte[] compressed,
bool tolerateTruncatedStream,
bool decompressConcatenated = false
)
{
using var stream = BZip2Stream.Create(
new MemoryStream(compressed),
CompressionMode.Decompress,
decompressConcatenated,
leaveOpen: false,
tolerateTruncatedStream: tolerateTruncatedStream
);
using var output = new MemoryStream();
stream.CopyTo(output);
return output.ToArray();
}
private static byte[] Concat(byte[] a, byte[] b)
{
var result = new byte[a.Length + b.Length];
Buffer.BlockCopy(a, 0, result, 0, a.Length);
Buffer.BlockCopy(b, 0, result, a.Length, b.Length);
return result;
}
private static byte[] Compress(string value)
{
using var memoryStream = new MemoryStream();