mirror of
https://github.com/adamhathcock/sharpcompress.git
synced 2026-09-23 15:34:34 +00:00
Zip ZStandard Writing with tests. Level support.
This commit is contained in:
@@ -28,4 +28,5 @@ public enum CompressionType
|
||||
Squashed,
|
||||
Crushed,
|
||||
Distilled,
|
||||
ZStandard,
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ internal enum ZipCompressionMethod
|
||||
Deflate64 = 9,
|
||||
BZip2 = 12,
|
||||
LZMA = 14,
|
||||
ZStd = 93,
|
||||
ZStandard = 93,
|
||||
Xz = 95,
|
||||
PPMd = 98,
|
||||
WinzipAes = 0x63, //http://www.winzip.com/aes_info.htm
|
||||
|
||||
@@ -46,6 +46,7 @@ public class ZipEntry : Entry
|
||||
ZipCompressionMethod.Reduce3 => CompressionType.Reduce3,
|
||||
ZipCompressionMethod.Reduce4 => CompressionType.Reduce4,
|
||||
ZipCompressionMethod.Explode => CompressionType.Explode,
|
||||
ZipCompressionMethod.ZStandard => CompressionType.ZStandard,
|
||||
_ => CompressionType.Unknown,
|
||||
};
|
||||
|
||||
|
||||
@@ -152,7 +152,7 @@ internal abstract class ZipFilePart : FilePart
|
||||
{
|
||||
return new XZStream(stream);
|
||||
}
|
||||
case ZipCompressionMethod.ZStd:
|
||||
case ZipCompressionMethod.ZStandard:
|
||||
{
|
||||
return new DecompressionStream(stream);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ public sealed class GZipWriter : AbstractWriter
|
||||
new GZipStream(
|
||||
destination,
|
||||
CompressionMode.Compress,
|
||||
options?.CompressionLevel ?? CompressionLevel.Default,
|
||||
(CompressionLevel)(options?.CompressionLevel ?? (int)CompressionLevel.Default),
|
||||
WriterOptions.ArchiveEncoding.GetEncoding()
|
||||
)
|
||||
);
|
||||
|
||||
@@ -1,24 +1,18 @@
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Deflate;
|
||||
using D = SharpCompress.Compressors.Deflate;
|
||||
|
||||
namespace SharpCompress.Writers.GZip;
|
||||
|
||||
public class GZipWriterOptions : WriterOptions
|
||||
{
|
||||
public GZipWriterOptions()
|
||||
: base(CompressionType.GZip) { }
|
||||
: base(CompressionType.GZip, (int)(D.CompressionLevel.Default)) { }
|
||||
|
||||
internal GZipWriterOptions(WriterOptions options)
|
||||
: base(options.CompressionType)
|
||||
: base(options.CompressionType, (int)(D.CompressionLevel.Default))
|
||||
{
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
|
||||
if (options is GZipWriterOptions writerOptions)
|
||||
{
|
||||
CompressionLevel = writerOptions.CompressionLevel;
|
||||
}
|
||||
CompressionLevel = options.CompressionLevel;
|
||||
}
|
||||
|
||||
public CompressionLevel CompressionLevel { get; set; } = CompressionLevel.Default;
|
||||
}
|
||||
|
||||
@@ -1,13 +1,41 @@
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Common;
|
||||
using D = SharpCompress.Compressors.Deflate;
|
||||
|
||||
namespace SharpCompress.Writers;
|
||||
|
||||
public class WriterOptions : OptionsBase
|
||||
{
|
||||
public WriterOptions(CompressionType compressionType) => CompressionType = compressionType;
|
||||
public WriterOptions(CompressionType compressionType)
|
||||
{
|
||||
CompressionType = compressionType;
|
||||
CompressionLevel = compressionType switch
|
||||
{
|
||||
CompressionType.ZStandard => 3,
|
||||
CompressionType.Deflate => (int)D.CompressionLevel.Default,
|
||||
CompressionType.Deflate64 => (int)D.CompressionLevel.Default,
|
||||
CompressionType.GZip => (int)D.CompressionLevel.Default,
|
||||
_ => 0
|
||||
};
|
||||
}
|
||||
|
||||
public WriterOptions(CompressionType compressionType, int compressionLevel)
|
||||
{
|
||||
CompressionType = compressionType;
|
||||
CompressionLevel = compressionLevel;
|
||||
}
|
||||
|
||||
public CompressionType CompressionType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The compression level to be used when the compression type supports variable levels.
|
||||
/// Valid ranges depend on the compression algorithm:
|
||||
/// - Deflate/GZip: 0-9 (0=no compression, 6=default, 9=best compression)
|
||||
/// - ZStandard: 1-22 (1=fastest, 3=default, 22=best compression)
|
||||
/// Note: BZip2 and LZMA do not support compression levels in this implementation.
|
||||
/// Defaults are set automatically based on compression type in the constructor.
|
||||
/// </summary>
|
||||
public int CompressionLevel { get; set; }
|
||||
|
||||
public static implicit operator WriterOptions(CompressionType compressionType) =>
|
||||
new(compressionType);
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ namespace SharpCompress.Writers.Zip;
|
||||
public class ZipWriter : AbstractWriter
|
||||
{
|
||||
private readonly CompressionType compressionType;
|
||||
private readonly CompressionLevel compressionLevel;
|
||||
private readonly int compressionLevel;
|
||||
private readonly List<ZipCentralDirectoryEntry> entries = new();
|
||||
private readonly string zipComment;
|
||||
private long streamPosition;
|
||||
@@ -36,7 +36,7 @@ public class ZipWriter : AbstractWriter
|
||||
}
|
||||
|
||||
compressionType = zipWriterOptions.CompressionType;
|
||||
compressionLevel = zipWriterOptions.DeflateCompressionLevel;
|
||||
compressionLevel = zipWriterOptions.CompressionLevel;
|
||||
|
||||
if (WriterOptions.LeaveStreamOpen)
|
||||
{
|
||||
@@ -69,6 +69,7 @@ public class ZipWriter : AbstractWriter
|
||||
CompressionType.BZip2 => ZipCompressionMethod.BZip2,
|
||||
CompressionType.LZMA => ZipCompressionMethod.LZMA,
|
||||
CompressionType.PPMd => ZipCompressionMethod.PPMd,
|
||||
CompressionType.ZStandard => ZipCompressionMethod.ZStandard,
|
||||
_ => throw new InvalidFormatException("Invalid compression method: " + compressionType),
|
||||
};
|
||||
|
||||
@@ -117,7 +118,7 @@ public class ZipWriter : AbstractWriter
|
||||
OutputStream.NotNull(),
|
||||
entry,
|
||||
compression,
|
||||
options.DeflateCompressionLevel ?? compressionLevel
|
||||
options.CompressionLevel ?? compressionLevel
|
||||
);
|
||||
}
|
||||
|
||||
@@ -312,7 +313,7 @@ public class ZipWriter : AbstractWriter
|
||||
private readonly Stream writeStream;
|
||||
private readonly ZipWriter writer;
|
||||
private readonly ZipCompressionMethod zipCompressionMethod;
|
||||
private readonly CompressionLevel compressionLevel;
|
||||
private readonly int compressionLevel;
|
||||
private SharpCompressStream? counting;
|
||||
private ulong decompressed;
|
||||
|
||||
@@ -325,7 +326,7 @@ public class ZipWriter : AbstractWriter
|
||||
Stream originalStream,
|
||||
ZipCentralDirectoryEntry entry,
|
||||
ZipCompressionMethod zipCompressionMethod,
|
||||
CompressionLevel compressionLevel
|
||||
int compressionLevel
|
||||
)
|
||||
{
|
||||
this.writer = writer;
|
||||
@@ -363,7 +364,7 @@ public class ZipWriter : AbstractWriter
|
||||
}
|
||||
case ZipCompressionMethod.Deflate:
|
||||
{
|
||||
return new DeflateStream(counting, CompressionMode.Compress, compressionLevel);
|
||||
return new DeflateStream(counting, CompressionMode.Compress, (CompressionLevel)compressionLevel);
|
||||
}
|
||||
case ZipCompressionMethod.BZip2:
|
||||
{
|
||||
@@ -389,6 +390,10 @@ public class ZipWriter : AbstractWriter
|
||||
counting.Write(writer.PpmdProperties.Properties, 0, 2);
|
||||
return new PpmdStream(writer.PpmdProperties, counting, true);
|
||||
}
|
||||
case ZipCompressionMethod.ZStandard:
|
||||
{
|
||||
return new ZstdSharp.CompressionStream(counting, (int)writer.WriterOptions.CompressionLevel);
|
||||
}
|
||||
default:
|
||||
{
|
||||
throw new NotSupportedException("CompressionMethod: " + zipCompressionMethod);
|
||||
|
||||
@@ -9,9 +9,29 @@ public class ZipWriterEntryOptions
|
||||
public CompressionType? CompressionType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When CompressionType.Deflate is used, this property is referenced. Defaults to CompressionLevel.Default.
|
||||
/// The compression level to be used when the compression type supports variable levels.
|
||||
/// Valid ranges depend on the compression algorithm:
|
||||
/// - Deflate/GZip: 0-9 (0=no compression, 6=default, 9=best compression)
|
||||
/// - ZStandard: 1-22 (1=fastest, 3=default, 22=best compression)
|
||||
/// When null, uses the archive's default compression level for the specified compression type.
|
||||
/// Note: BZip2 and LZMA do not support compression levels in this implementation.
|
||||
/// </summary>
|
||||
public CompressionLevel? DeflateCompressionLevel { get; set; }
|
||||
public int? CompressionLevel { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When CompressionType.Deflate is used, this property is referenced.
|
||||
/// Valid range: 0-9 (0=no compression, 6=default, 9=best compression).
|
||||
/// When null, uses the archive's default compression level.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This property is deprecated. Use <see cref="CompressionLevel"/> instead.
|
||||
/// </remarks>
|
||||
[Obsolete("Use CompressionLevel property instead. This property will be removed in a future version.")]
|
||||
public CompressionLevel? DeflateCompressionLevel
|
||||
{
|
||||
get => CompressionLevel.HasValue ? (CompressionLevel)Math.Min(CompressionLevel.Value, 9) : null;
|
||||
set => CompressionLevel = value.HasValue ? (int)value.Value : null;
|
||||
}
|
||||
|
||||
public string? EntryComment { get; set; }
|
||||
|
||||
|
||||
@@ -1,31 +1,66 @@
|
||||
using System;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Deflate;
|
||||
using D = SharpCompress.Compressors.Deflate;
|
||||
|
||||
namespace SharpCompress.Writers.Zip;
|
||||
|
||||
public class ZipWriterOptions : WriterOptions
|
||||
{
|
||||
public ZipWriterOptions(CompressionType compressionType)
|
||||
: base(compressionType) { }
|
||||
public ZipWriterOptions(CompressionType compressionType, CompressionLevel compressionLevel = D.CompressionLevel.Default)
|
||||
: base(compressionType, (int)compressionLevel) { }
|
||||
|
||||
internal ZipWriterOptions(WriterOptions options)
|
||||
: base(options.CompressionType)
|
||||
{
|
||||
LeaveStreamOpen = options.LeaveStreamOpen;
|
||||
ArchiveEncoding = options.ArchiveEncoding;
|
||||
CompressionLevel = options.CompressionLevel;
|
||||
|
||||
if (options is ZipWriterOptions writerOptions)
|
||||
{
|
||||
UseZip64 = writerOptions.UseZip64;
|
||||
DeflateCompressionLevel = writerOptions.DeflateCompressionLevel;
|
||||
ArchiveComment = writerOptions.ArchiveComment;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When CompressionType.Deflate is used, this property is referenced. Defaults to CompressionLevel.Default.
|
||||
/// Sets the compression level for Deflate compression (0-9).
|
||||
/// This is a convenience method that sets the CompressionLevel property for Deflate compression.
|
||||
/// </summary>
|
||||
public CompressionLevel DeflateCompressionLevel { get; set; } = CompressionLevel.Default;
|
||||
/// <param name="level">Deflate compression level (0=no compression, 6=default, 9=best compression)</param>
|
||||
public void SetDeflateCompressionLevel(CompressionLevel level)
|
||||
{
|
||||
CompressionLevel = (int)level;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the compression level for ZStandard compression (1-22).
|
||||
/// This is a convenience method that sets the CompressionLevel property for ZStandard compression.
|
||||
/// </summary>
|
||||
/// <param name="level">ZStandard compression level (1=fastest, 3=default, 22=best compression)</param>
|
||||
/// <exception cref="ArgumentOutOfRangeException">Thrown when level is not between 1 and 22</exception>
|
||||
public void SetZStandardCompressionLevel(int level)
|
||||
{
|
||||
if (level < 1 || level > 22)
|
||||
throw new ArgumentOutOfRangeException(nameof(level), "ZStandard compression level must be between 1 and 22");
|
||||
|
||||
CompressionLevel = level;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Legacy property for Deflate compression levels.
|
||||
/// Valid range: 0-9 (0=no compression, 6=default, 9=best compression).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This property is deprecated. Use <see cref="WriterOptions.CompressionLevel"/> or <see cref="SetDeflateCompressionLevel"/> instead.
|
||||
/// </remarks>
|
||||
[Obsolete("Use CompressionLevel property or SetDeflateCompressionLevel method instead. This property will be removed in a future version.")]
|
||||
public CompressionLevel DeflateCompressionLevel
|
||||
{
|
||||
get => (CompressionLevel)Math.Min(CompressionLevel, 9);
|
||||
set => CompressionLevel = (int)value;
|
||||
}
|
||||
|
||||
public string? ArchiveComment { get; set; }
|
||||
|
||||
|
||||
@@ -4,8 +4,12 @@ using System.IO;
|
||||
using System.Linq;
|
||||
using SharpCompress.Archives;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Xz;
|
||||
using SharpCompress.Crypto;
|
||||
using SharpCompress.IO;
|
||||
using SharpCompress.Readers;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test;
|
||||
@@ -339,4 +343,182 @@ public class ArchiveTests : ReaderTests
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates CRC32 for the given data using SharpCompress implementation
|
||||
/// </summary>
|
||||
protected static uint CalculateCrc32(byte[] data) => Crc32.Compute(data);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a writer with the specified compression type and level
|
||||
/// </summary>
|
||||
protected static IWriter CreateWriterWithLevel(Stream stream, CompressionType compressionType, int? compressionLevel = null)
|
||||
{
|
||||
var writerOptions = new ZipWriterOptions(compressionType);
|
||||
if (compressionLevel.HasValue)
|
||||
{
|
||||
writerOptions.CompressionLevel = compressionLevel.Value;
|
||||
}
|
||||
return WriterFactory.Open(stream, ArchiveType.Zip, writerOptions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies archive content against expected files with CRC32 validation
|
||||
/// </summary>
|
||||
protected void VerifyArchiveContent(MemoryStream zipStream, Dictionary<string, (byte[] data, uint crc)> expectedFiles)
|
||||
{
|
||||
zipStream.Position = 0;
|
||||
using var archive = ArchiveFactory.Open(zipStream);
|
||||
Assert.Equal(expectedFiles.Count, archive.Entries.Count());
|
||||
|
||||
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
|
||||
{
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
entryStream.CopyTo(extractedStream);
|
||||
var extractedData = extractedStream.ToArray();
|
||||
|
||||
Assert.True(expectedFiles.ContainsKey(entry.Key.NotNull()), $"Unexpected entry: {entry.Key}");
|
||||
|
||||
var (expectedData, expectedCrc) = expectedFiles[entry.Key.NotNull()];
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(expectedData.Length, extractedData.Length);
|
||||
|
||||
// For large files, spot check rather than full comparison for performance
|
||||
if (expectedData.Length > 1024 * 1024)
|
||||
{
|
||||
VerifyDataSpotCheck(expectedData, extractedData);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(expectedData, extractedData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs efficient spot checks on large data arrays
|
||||
/// </summary>
|
||||
protected static void VerifyDataSpotCheck(byte[] expected, byte[] actual)
|
||||
{
|
||||
// Check first, middle, and last 1KB
|
||||
Assert.Equal(expected.Take(1024), actual.Take(1024));
|
||||
var mid = expected.Length / 2;
|
||||
Assert.Equal(expected.Skip(mid).Take(1024), actual.Skip(mid).Take(1024));
|
||||
Assert.Equal(expected.Skip(Math.Max(0, expected.Length - 1024)), actual.Skip(Math.Max(0, actual.Length - 1024)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies compression ratio meets expectations
|
||||
/// </summary>
|
||||
protected void VerifyCompressionRatio(long originalSize, long compressedSize, double maxRatio, string context)
|
||||
{
|
||||
var compressionRatio = (double)compressedSize / originalSize;
|
||||
Assert.True(compressionRatio < maxRatio,
|
||||
$"Expected better compression for {context}. Original: {originalSize}, Compressed: {compressedSize}, Ratio: {compressionRatio:P}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a memory-based archive with specified files and compression
|
||||
/// </summary>
|
||||
protected MemoryStream CreateMemoryArchive(Dictionary<string, byte[]> files, CompressionType compressionType, int? compressionLevel = null)
|
||||
{
|
||||
var zipStream = new MemoryStream();
|
||||
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
|
||||
{
|
||||
foreach (var kvp in files)
|
||||
{
|
||||
writer.Write(kvp.Key, new MemoryStream(kvp.Value));
|
||||
}
|
||||
}
|
||||
return zipStream;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies streaming CRC calculation for large data
|
||||
/// </summary>
|
||||
protected void VerifyStreamingCrc(Stream entryStream, uint expectedCrc, long expectedLength)
|
||||
{
|
||||
using var crcStream = new Crc32Stream(Stream.Null);
|
||||
const int bufferSize = 64 * 1024;
|
||||
var buffer = new byte[bufferSize];
|
||||
int totalBytesRead = 0;
|
||||
int bytesRead;
|
||||
|
||||
while ((bytesRead = entryStream.Read(buffer, 0, bufferSize)) > 0)
|
||||
{
|
||||
crcStream.Write(buffer, 0, bytesRead);
|
||||
totalBytesRead += bytesRead;
|
||||
}
|
||||
|
||||
var actualCrc = crcStream.Crc;
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(expectedLength, totalBytesRead);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates and verifies a basic archive with compression testing
|
||||
/// </summary>
|
||||
protected void CreateAndVerifyBasicArchive(
|
||||
Dictionary<string, byte[]> testFiles,
|
||||
CompressionType compressionType,
|
||||
int? compressionLevel = null,
|
||||
double maxCompressionRatio = 0.8)
|
||||
{
|
||||
// Calculate expected CRCs
|
||||
var expectedFiles = testFiles.ToDictionary(
|
||||
kvp => kvp.Key,
|
||||
kvp => (data: kvp.Value, crc: CalculateCrc32(kvp.Value))
|
||||
);
|
||||
|
||||
// Create archive
|
||||
using var zipStream = CreateMemoryArchive(testFiles, compressionType, compressionLevel);
|
||||
|
||||
// Verify compression occurred if expected
|
||||
if (compressionType != CompressionType.None)
|
||||
{
|
||||
var originalSize = testFiles.Values.Sum(data => (long)data.Length);
|
||||
VerifyCompressionRatio(originalSize, zipStream.Length, maxCompressionRatio, compressionType.ToString());
|
||||
}
|
||||
|
||||
// Verify content
|
||||
VerifyArchiveContent(zipStream, expectedFiles);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies archive entries have correct compression type
|
||||
/// </summary>
|
||||
protected void VerifyCompressionType(MemoryStream zipStream, CompressionType expectedCompressionType)
|
||||
{
|
||||
zipStream.Position = 0;
|
||||
using var archive = ArchiveFactory.Open(zipStream);
|
||||
|
||||
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
|
||||
{
|
||||
Assert.Equal(expectedCompressionType, entry.CompressionType);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts and verifies a single entry from archive
|
||||
/// </summary>
|
||||
protected (byte[] data, uint crc) ExtractAndVerifyEntry(MemoryStream zipStream, string entryName)
|
||||
{
|
||||
zipStream.Position = 0;
|
||||
using var archive = ArchiveFactory.Open(zipStream);
|
||||
|
||||
var entry = archive.Entries.FirstOrDefault(e => e.Key == entryName && !e.IsDirectory);
|
||||
Assert.NotNull(entry);
|
||||
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
entryStream.CopyTo(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var crc = CalculateCrc32(extractedData);
|
||||
|
||||
return (extractedData, crc);
|
||||
}
|
||||
}
|
||||
|
||||
97
tests/SharpCompress.Test/Zip/TestPseudoTextStream.cs
Normal file
97
tests/SharpCompress.Test/Zip/TestPseudoTextStream.cs
Normal file
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
/// <summary>
|
||||
/// Generates pseudo English-style text for testing - Nanook
|
||||
/// </summary>
|
||||
internal class TestPseudoTextStream : Stream
|
||||
{
|
||||
private static readonly char[] _vowels = { 'a', 'e', 'i', 'o', 'u' };
|
||||
private static readonly char[] _consonants = "bcdfghjklmnpqrstvwxyz".ToCharArray();
|
||||
|
||||
private long _position = 0;
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position
|
||||
{
|
||||
get => _position;
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public override void Flush() => throw new NotSupportedException();
|
||||
|
||||
public static byte[] Create(int size)
|
||||
{
|
||||
byte[] data = new byte[size];
|
||||
using (TestPseudoTextStream rts = new TestPseudoTextStream())
|
||||
{
|
||||
int bufferSize = 64 * 1024; // 64k blocks
|
||||
byte[] buffer = new byte[bufferSize];
|
||||
int bytesRead = 0;
|
||||
int totalBytesRead = 0;
|
||||
|
||||
while (totalBytesRead < size)
|
||||
{
|
||||
bytesRead = rts.Read(buffer, 0, Math.Min(bufferSize, size - totalBytesRead));
|
||||
Array.Copy(buffer, 0, data, totalBytesRead, bytesRead);
|
||||
totalBytesRead += bytesRead;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
int bytesRead = 0;
|
||||
while (bytesRead < count)
|
||||
{
|
||||
string word = GenerateDeterministicWord(_position + bytesRead);
|
||||
byte[] wordBytes = System.Text.Encoding.ASCII.GetBytes(word + " ");
|
||||
|
||||
int bytesToCopy = Math.Min(wordBytes.Length, count - bytesRead);
|
||||
Array.Copy(wordBytes, 0, buffer, offset + bytesRead, bytesToCopy);
|
||||
|
||||
bytesRead += bytesToCopy;
|
||||
_position += bytesToCopy;
|
||||
}
|
||||
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
private string GenerateDeterministicWord(long seed)
|
||||
{
|
||||
int length = (int)(seed % 7) + 2; // 2 to 8 letters
|
||||
int vowelCount = (int)((seed / 7) % 4) + 2; // 2 to 5 vowels
|
||||
|
||||
System.Text.StringBuilder word = new System.Text.StringBuilder(length);
|
||||
int vowelsAdded = 0;
|
||||
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
if (vowelsAdded < vowelCount && ((seed >> i) & 1) == 0)
|
||||
{
|
||||
word.Append(_vowels[(int)(seed >> (i + 1)) % _vowels.Length]);
|
||||
vowelsAdded++;
|
||||
}
|
||||
else
|
||||
{
|
||||
word.Append(_consonants[(int)(seed >> (i + 1)) % _consonants.Length]);
|
||||
}
|
||||
}
|
||||
|
||||
return word.ToString();
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
}
|
||||
194
tests/SharpCompress.Test/Zip/ZipMemoryArchiveWithCrcTests.cs
Normal file
194
tests/SharpCompress.Test/Zip/ZipMemoryArchiveWithCrcTests.cs
Normal file
@@ -0,0 +1,194 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using SharpCompress.Archives.Zip;
|
||||
using SharpCompress.Common;
|
||||
using SharpCompress.Compressors.Deflate;
|
||||
using SharpCompress.Compressors.Xz;
|
||||
using SharpCompress.Crypto;
|
||||
using SharpCompress.Writers;
|
||||
using SharpCompress.Writers.Zip;
|
||||
using Xunit;
|
||||
|
||||
namespace SharpCompress.Test.Zip;
|
||||
|
||||
public class ZipTypesLevelsWithCrcRatioTests : ArchiveTests
|
||||
{
|
||||
public ZipTypesLevelsWithCrcRatioTests() => UseExtensionInsteadOfNameToVerify = true;
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 1, 0.11f)] // was 0.8f, actual 0.104
|
||||
[InlineData(CompressionType.Deflate, 3, 1, 0.08f)] // was 0.8f, actual 0.078
|
||||
[InlineData(CompressionType.Deflate, 6, 1, 0.05f)] // was 0.8f, actual ~0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 1, 0.04f)] // was 0.7f, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 1, 1, 0.025f)] // was 0.8f, actual 0.023
|
||||
[InlineData(CompressionType.ZStandard, 3, 1, 0.015f)] // was 0.7f, actual 0.013
|
||||
[InlineData(CompressionType.ZStandard, 9, 1, 0.006f)] // was 0.7f, actual 0.005
|
||||
[InlineData(CompressionType.ZStandard, 22, 1, 0.005f)] // was 0.7f, actual 0.004
|
||||
[InlineData(CompressionType.BZip2, 0, 1, 0.035f)] // was 0.8f, actual 0.033
|
||||
[InlineData(CompressionType.LZMA, 0, 1, 0.005f)] // was 0.8f, actual 0.004
|
||||
[InlineData(CompressionType.None, 0, 1, 1.001f)] // was 1.1f, actual 1.000
|
||||
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8f, actual 0.042
|
||||
[InlineData(CompressionType.ZStandard, 3, 2, 0.012f)] // was 0.7f, actual 0.010
|
||||
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8f, actual 0.032
|
||||
[InlineData(CompressionType.Deflate, 9, 3, 0.04f)] // was 0.7f, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 9, 3, 0.003f)] // was 0.7f, actual 0.002
|
||||
public void Zip_Create_Archive_With_3_Files_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
|
||||
{
|
||||
const int OneMiB = 1024 * 1024;
|
||||
var baseSize = sizeMb * OneMiB;
|
||||
|
||||
// Generate test content for files with sizes based on the sizeMb parameter
|
||||
var file1Data = TestPseudoTextStream.Create(baseSize);
|
||||
var file2Data = TestPseudoTextStream.Create(baseSize * 2);
|
||||
var file3Data = TestPseudoTextStream.Create(baseSize * 3);
|
||||
|
||||
var expectedFiles = new Dictionary<string, (byte[] data, uint crc)>
|
||||
{
|
||||
[$"file1_{sizeMb}MiB.txt"] = (file1Data, CalculateCrc32(file1Data)),
|
||||
[$"data/file2_{sizeMb * 2}MiB.txt"] = (file2Data, CalculateCrc32(file2Data)),
|
||||
[$"deep/nested/file3_{sizeMb * 3}MiB.txt"] = (file3Data, CalculateCrc32(file3Data))
|
||||
};
|
||||
|
||||
// Create zip archive in memory
|
||||
using var zipStream = new MemoryStream();
|
||||
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
|
||||
{
|
||||
writer.Write($"file1_{sizeMb}MiB.txt", new MemoryStream(file1Data));
|
||||
writer.Write($"data/file2_{sizeMb * 2}MiB.txt", new MemoryStream(file2Data));
|
||||
writer.Write($"deep/nested/file3_{sizeMb * 3}MiB.txt", new MemoryStream(file3Data));
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var originalSize = file1Data.Length + file2Data.Length + file3Data.Length;
|
||||
var actualRatio = (double)zipStream.Length / originalSize;
|
||||
//Debug.WriteLine($"Zip_Create_Archive_With_3_Files_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
// Verify compression occurred (except for None compression type)
|
||||
if (compressionType != CompressionType.None)
|
||||
{
|
||||
Assert.True(zipStream.Length < originalSize, $"Compression failed: compressed={zipStream.Length}, original={originalSize}");
|
||||
}
|
||||
|
||||
// Verify compression ratio
|
||||
VerifyCompressionRatio(originalSize, zipStream.Length, expectedRatio, $"{compressionType} level {compressionLevel}");
|
||||
|
||||
// Verify archive content and CRC32
|
||||
VerifyArchiveContent(zipStream, expectedFiles);
|
||||
|
||||
// Verify compression type is correctly set
|
||||
VerifyCompressionType(zipStream, compressionType);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 4, 0.11f)] // was 0.8, actual 0.105
|
||||
[InlineData(CompressionType.Deflate, 3, 4, 0.08f)] // was 0.8, actual 0.077
|
||||
[InlineData(CompressionType.Deflate, 6, 4, 0.045f)] // was 0.8, actual 0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 4, 0.04f)] // was 0.8, actual 0.037
|
||||
[InlineData(CompressionType.ZStandard, 1, 4, 0.025f)] // was 0.8, actual 0.022
|
||||
[InlineData(CompressionType.ZStandard, 3, 4, 0.012f)] // was 0.8, actual 0.010
|
||||
[InlineData(CompressionType.ZStandard, 9, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
[InlineData(CompressionType.ZStandard, 22, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
[InlineData(CompressionType.BZip2, 0, 4, 0.035f)] // was 0.8, actual 0.032
|
||||
[InlineData(CompressionType.LZMA, 0, 4, 0.003f)] // was 0.8, actual 0.002
|
||||
public void Zip_WriterFactory_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
|
||||
{
|
||||
var fileSize = sizeMb * 1024 * 1024;
|
||||
|
||||
var testData = TestPseudoTextStream.Create(fileSize);
|
||||
var expectedCrc = CalculateCrc32(testData);
|
||||
|
||||
// Create archive with specified compression level
|
||||
using var zipStream = new MemoryStream();
|
||||
var writerOptions = new ZipWriterOptions(compressionType) { CompressionLevel = compressionLevel };
|
||||
|
||||
using (var writer = WriterFactory.Open(zipStream, ArchiveType.Zip, writerOptions))
|
||||
{
|
||||
writer.Write($"{compressionType}_level_{compressionLevel}_{sizeMb}MiB.txt", new MemoryStream(testData));
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var actualRatio = (double)zipStream.Length / testData.Length;
|
||||
//Debug.WriteLine($"Zip_WriterFactory_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
VerifyCompressionRatio(testData.Length, zipStream.Length, expectedRatio, $"{compressionType} level {compressionLevel}");
|
||||
|
||||
// Verify the archive
|
||||
zipStream.Position = 0;
|
||||
using var archive = ZipArchive.Open(zipStream);
|
||||
|
||||
var entry = archive.Entries.Single(e => !e.IsDirectory);
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
entryStream.CopyTo(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(compressionType, entry.CompressionType);
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(testData.Length, extractedData.Length);
|
||||
Assert.Equal(testData, extractedData);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(CompressionType.Deflate, 1, 2, 0.11f)] // was 0.8, actual 0.104
|
||||
[InlineData(CompressionType.Deflate, 3, 2, 0.08f)] // was 0.8, actual 0.077
|
||||
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8, actual 0.042
|
||||
[InlineData(CompressionType.Deflate, 9, 2, 0.04f)] // was 0.7, actual 0.038
|
||||
[InlineData(CompressionType.ZStandard, 1, 2, 0.025f)] // was 0.8, actual 0.023
|
||||
[InlineData(CompressionType.ZStandard, 3, 2, 0.015f)] // was 0.7, actual 0.012
|
||||
[InlineData(CompressionType.ZStandard, 9, 2, 0.006f)] // was 0.7, actual 0.005
|
||||
[InlineData(CompressionType.ZStandard, 22, 2, 0.005f)] // was 0.7, actual 0.004
|
||||
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8, actual 0.032
|
||||
[InlineData(CompressionType.LZMA, 0, 2, 0.005f)] // was 0.8, actual 0.004
|
||||
public void Zip_ZipArchiveOpen_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
|
||||
{
|
||||
var fileSize = sizeMb * 1024 * 1024;
|
||||
|
||||
var testData = TestPseudoTextStream.Create(fileSize);
|
||||
var expectedCrc = CalculateCrc32(testData);
|
||||
|
||||
// Create archive with specified compression and level
|
||||
using var zipStream = new MemoryStream();
|
||||
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
|
||||
{
|
||||
writer.Write($"{compressionType}_{compressionLevel}_{sizeMb}MiB.txt", new MemoryStream(testData));
|
||||
}
|
||||
|
||||
// Calculate and output actual compression ratio
|
||||
var actualRatio = (double)zipStream.Length / testData.Length;
|
||||
//Debug.WriteLine($"Zip_ZipArchiveOpen_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
|
||||
|
||||
// Verify the archive
|
||||
zipStream.Position = 0;
|
||||
using var archive = ZipArchive.Open(zipStream);
|
||||
|
||||
var entry = archive.Entries.Single(e => !e.IsDirectory);
|
||||
using var entryStream = entry.OpenEntryStream();
|
||||
using var extractedStream = new MemoryStream();
|
||||
entryStream.CopyTo(extractedStream);
|
||||
|
||||
var extractedData = extractedStream.ToArray();
|
||||
var actualCrc = CalculateCrc32(extractedData);
|
||||
|
||||
Assert.Equal(compressionType, entry.CompressionType);
|
||||
Assert.Equal(expectedCrc, actualCrc);
|
||||
Assert.Equal(testData.Length, extractedData.Length);
|
||||
|
||||
// For smaller files, verify full content; for larger, spot check
|
||||
if (testData.Length <= sizeMb * 2)
|
||||
{
|
||||
Assert.Equal(testData, extractedData);
|
||||
}
|
||||
else
|
||||
{
|
||||
VerifyDataSpotCheck(testData, extractedData);
|
||||
}
|
||||
|
||||
VerifyCompressionRatio(testData.Length, zipStream.Length, expectedRatio, $"{compressionType} Level {compressionLevel}");
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user