starred/sharpcompress
1
0
Fork 0
mirror of https://github.com/adamhathcock/sharpcompress.git synced 2026-09-22 23:15:20 +00:00
Code Issues 975 Packages Projects Releases 25 Wiki Activity

Zip ZStandard Writing with tests. Level support.

Browse Source
This commit is contained in:
Nanook
2025-07-24 00:23:55 +01:00
parent 624c385e27
commit c5de3d8cc1
13 changed files with 585 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
tests/SharpCompress.Test/ArchiveTests.cs
View File

@@ -4,8 +4,12 @@ using System.IO;
using System.Linq;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Compressors.Xz;
using SharpCompress.Crypto;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Writers;
using SharpCompress.Writers.Zip;
using Xunit;
namespace SharpCompress.Test;
@@ -339,4 +343,182 @@ public class ArchiveTests : ReaderTests
}
}
}
/// <summary>
/// Calculates CRC32 for the given data using SharpCompress implementation
/// </summary>
protected static uint CalculateCrc32(byte[] data) => Crc32.Compute(data);
/// <summary>
/// Creates a writer with the specified compression type and level
/// </summary>
protected static IWriter CreateWriterWithLevel(Stream stream, CompressionType compressionType, int? compressionLevel = null)
{
var writerOptions = new ZipWriterOptions(compressionType);
if (compressionLevel.HasValue)
{
writerOptions.CompressionLevel = compressionLevel.Value;
}
return WriterFactory.Open(stream, ArchiveType.Zip, writerOptions);
}
/// <summary>
/// Verifies archive content against expected files with CRC32 validation
/// </summary>
protected void VerifyArchiveContent(MemoryStream zipStream, Dictionary<string, (byte[] data, uint crc)> expectedFiles)
{
zipStream.Position = 0;
using var archive = ArchiveFactory.Open(zipStream);
Assert.Equal(expectedFiles.Count, archive.Entries.Count());
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
{
using var entryStream = entry.OpenEntryStream();
using var extractedStream = new MemoryStream();
entryStream.CopyTo(extractedStream);
var extractedData = extractedStream.ToArray();
Assert.True(expectedFiles.ContainsKey(entry.Key.NotNull()), $"Unexpected entry: {entry.Key}");
var (expectedData, expectedCrc) = expectedFiles[entry.Key.NotNull()];
var actualCrc = CalculateCrc32(extractedData);
Assert.Equal(expectedCrc, actualCrc);
Assert.Equal(expectedData.Length, extractedData.Length);
// For large files, spot check rather than full comparison for performance
if (expectedData.Length > 1024 * 1024)
{
VerifyDataSpotCheck(expectedData, extractedData);
}
else
{
Assert.Equal(expectedData, extractedData);
}
}
}
/// <summary>
/// Performs efficient spot checks on large data arrays
/// </summary>
protected static void VerifyDataSpotCheck(byte[] expected, byte[] actual)
{
// Check first, middle, and last 1KB
Assert.Equal(expected.Take(1024), actual.Take(1024));
var mid = expected.Length / 2;
Assert.Equal(expected.Skip(mid).Take(1024), actual.Skip(mid).Take(1024));
Assert.Equal(expected.Skip(Math.Max(0, expected.Length - 1024)), actual.Skip(Math.Max(0, actual.Length - 1024)));
}
/// <summary>
/// Verifies compression ratio meets expectations
/// </summary>
protected void VerifyCompressionRatio(long originalSize, long compressedSize, double maxRatio, string context)
{
var compressionRatio = (double)compressedSize / originalSize;
Assert.True(compressionRatio < maxRatio,
$"Expected better compression for {context}. Original: {originalSize}, Compressed: {compressedSize}, Ratio: {compressionRatio:P}");
}
/// <summary>
/// Creates a memory-based archive with specified files and compression
/// </summary>
protected MemoryStream CreateMemoryArchive(Dictionary<string, byte[]> files, CompressionType compressionType, int? compressionLevel = null)
{
var zipStream = new MemoryStream();
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
{
foreach (var kvp in files)
{
writer.Write(kvp.Key, new MemoryStream(kvp.Value));
}
}
return zipStream;
}
/// <summary>
/// Verifies streaming CRC calculation for large data
/// </summary>
protected void VerifyStreamingCrc(Stream entryStream, uint expectedCrc, long expectedLength)
{
using var crcStream = new Crc32Stream(Stream.Null);
const int bufferSize = 64 * 1024;
var buffer = new byte[bufferSize];
int totalBytesRead = 0;
int bytesRead;
while ((bytesRead = entryStream.Read(buffer, 0, bufferSize)) > 0)
{
crcStream.Write(buffer, 0, bytesRead);
totalBytesRead += bytesRead;
}
var actualCrc = crcStream.Crc;
Assert.Equal(expectedCrc, actualCrc);
Assert.Equal(expectedLength, totalBytesRead);
}
/// <summary>
/// Creates and verifies a basic archive with compression testing
/// </summary>
protected void CreateAndVerifyBasicArchive(
Dictionary<string, byte[]> testFiles,
CompressionType compressionType,
int? compressionLevel = null,
double maxCompressionRatio = 0.8)
{
// Calculate expected CRCs
var expectedFiles = testFiles.ToDictionary(
kvp => kvp.Key,
kvp => (data: kvp.Value, crc: CalculateCrc32(kvp.Value))
);
// Create archive
using var zipStream = CreateMemoryArchive(testFiles, compressionType, compressionLevel);
// Verify compression occurred if expected
if (compressionType != CompressionType.None)
{
var originalSize = testFiles.Values.Sum(data => (long)data.Length);
VerifyCompressionRatio(originalSize, zipStream.Length, maxCompressionRatio, compressionType.ToString());
}
// Verify content
VerifyArchiveContent(zipStream, expectedFiles);
}
/// <summary>
/// Verifies archive entries have correct compression type
/// </summary>
protected void VerifyCompressionType(MemoryStream zipStream, CompressionType expectedCompressionType)
{
zipStream.Position = 0;
using var archive = ArchiveFactory.Open(zipStream);
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
{
Assert.Equal(expectedCompressionType, entry.CompressionType);
}
}
/// <summary>
/// Extracts and verifies a single entry from archive
/// </summary>
protected (byte[] data, uint crc) ExtractAndVerifyEntry(MemoryStream zipStream, string entryName)
{
zipStream.Position = 0;
using var archive = ArchiveFactory.Open(zipStream);
var entry = archive.Entries.FirstOrDefault(e => e.Key == entryName && !e.IsDirectory);
Assert.NotNull(entry);
using var entryStream = entry.OpenEntryStream();
using var extractedStream = new MemoryStream();
entryStream.CopyTo(extractedStream);
var extractedData = extractedStream.ToArray();
var crc = CalculateCrc32(extractedData);
return (extractedData, crc);
}
}

97
tests/SharpCompress.Test/Zip/TestPseudoTextStream.cs Normal file
View File

@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace SharpCompress.Test.Zip;
/// <summary>
/// Generates pseudo English-style text for testing - Nanook
/// </summary>
internal class TestPseudoTextStream : Stream
{
private static readonly char[] _vowels = { 'a', 'e', 'i', 'o', 'u' };
private static readonly char[] _consonants = "bcdfghjklmnpqrstvwxyz".ToCharArray();
private long _position = 0;
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => _position;
set => throw new NotSupportedException();
}
public override void Flush() => throw new NotSupportedException();
public static byte[] Create(int size)
{
byte[] data = new byte[size];
using (TestPseudoTextStream rts = new TestPseudoTextStream())
{
int bufferSize = 64 * 1024; // 64k blocks
byte[] buffer = new byte[bufferSize];
int bytesRead = 0;
int totalBytesRead = 0;
while (totalBytesRead < size)
{
bytesRead = rts.Read(buffer, 0, Math.Min(bufferSize, size - totalBytesRead));
Array.Copy(buffer, 0, data, totalBytesRead, bytesRead);
totalBytesRead += bytesRead;
}
}
return data;
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = 0;
while (bytesRead < count)
{
string word = GenerateDeterministicWord(_position + bytesRead);
byte[] wordBytes = System.Text.Encoding.ASCII.GetBytes(word + " ");
int bytesToCopy = Math.Min(wordBytes.Length, count - bytesRead);
Array.Copy(wordBytes, 0, buffer, offset + bytesRead, bytesToCopy);
bytesRead += bytesToCopy;
_position += bytesToCopy;
}
return bytesRead;
}
private string GenerateDeterministicWord(long seed)
{
int length = (int)(seed % 7) + 2; // 2 to 8 letters
int vowelCount = (int)((seed / 7) % 4) + 2; // 2 to 5 vowels
System.Text.StringBuilder word = new System.Text.StringBuilder(length);
int vowelsAdded = 0;
for (int i = 0; i < length; i++)
{
if (vowelsAdded < vowelCount && ((seed >> i) & 1) == 0)
{
word.Append(_vowels[(int)(seed >> (i + 1)) % _vowels.Length]);
vowelsAdded++;
}
else
{
word.Append(_consonants[(int)(seed >> (i + 1)) % _consonants.Length]);
}
}
return word.ToString();
}
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
}

194
tests/SharpCompress.Test/Zip/ZipMemoryArchiveWithCrcTests.cs Normal file
View File

@@ -0,0 +1,194 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
using SharpCompress.Compressors.Deflate;
using SharpCompress.Compressors.Xz;
using SharpCompress.Crypto;
using SharpCompress.Writers;
using SharpCompress.Writers.Zip;
using Xunit;
namespace SharpCompress.Test.Zip;
public class ZipTypesLevelsWithCrcRatioTests : ArchiveTests
{
public ZipTypesLevelsWithCrcRatioTests() => UseExtensionInsteadOfNameToVerify = true;
[Theory]
[InlineData(CompressionType.Deflate, 1, 1, 0.11f)] // was 0.8f, actual 0.104
[InlineData(CompressionType.Deflate, 3, 1, 0.08f)] // was 0.8f, actual 0.078
[InlineData(CompressionType.Deflate, 6, 1, 0.05f)] // was 0.8f, actual ~0.042
[InlineData(CompressionType.Deflate, 9, 1, 0.04f)] // was 0.7f, actual 0.038
[InlineData(CompressionType.ZStandard, 1, 1, 0.025f)] // was 0.8f, actual 0.023
[InlineData(CompressionType.ZStandard, 3, 1, 0.015f)] // was 0.7f, actual 0.013
[InlineData(CompressionType.ZStandard, 9, 1, 0.006f)] // was 0.7f, actual 0.005
[InlineData(CompressionType.ZStandard, 22, 1, 0.005f)] // was 0.7f, actual 0.004
[InlineData(CompressionType.BZip2, 0, 1, 0.035f)] // was 0.8f, actual 0.033
[InlineData(CompressionType.LZMA, 0, 1, 0.005f)] // was 0.8f, actual 0.004
[InlineData(CompressionType.None, 0, 1, 1.001f)] // was 1.1f, actual 1.000
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8f, actual 0.042
[InlineData(CompressionType.ZStandard, 3, 2, 0.012f)] // was 0.7f, actual 0.010
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8f, actual 0.032
[InlineData(CompressionType.Deflate, 9, 3, 0.04f)] // was 0.7f, actual 0.038
[InlineData(CompressionType.ZStandard, 9, 3, 0.003f)] // was 0.7f, actual 0.002
public void Zip_Create_Archive_With_3_Files_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
{
const int OneMiB = 1024 * 1024;
var baseSize = sizeMb * OneMiB;
// Generate test content for files with sizes based on the sizeMb parameter
var file1Data = TestPseudoTextStream.Create(baseSize);
var file2Data = TestPseudoTextStream.Create(baseSize * 2);
var file3Data = TestPseudoTextStream.Create(baseSize * 3);
var expectedFiles = new Dictionary<string, (byte[] data, uint crc)>
{
[$"file1_{sizeMb}MiB.txt"] = (file1Data, CalculateCrc32(file1Data)),
[$"data/file2_{sizeMb * 2}MiB.txt"] = (file2Data, CalculateCrc32(file2Data)),
[$"deep/nested/file3_{sizeMb * 3}MiB.txt"] = (file3Data, CalculateCrc32(file3Data))
};
// Create zip archive in memory
using var zipStream = new MemoryStream();
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
{
writer.Write($"file1_{sizeMb}MiB.txt", new MemoryStream(file1Data));
writer.Write($"data/file2_{sizeMb * 2}MiB.txt", new MemoryStream(file2Data));
writer.Write($"deep/nested/file3_{sizeMb * 3}MiB.txt", new MemoryStream(file3Data));
}
// Calculate and output actual compression ratio
var originalSize = file1Data.Length + file2Data.Length + file3Data.Length;
var actualRatio = (double)zipStream.Length / originalSize;
//Debug.WriteLine($"Zip_Create_Archive_With_3_Files_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
// Verify compression occurred (except for None compression type)
if (compressionType != CompressionType.None)
{
Assert.True(zipStream.Length < originalSize, $"Compression failed: compressed={zipStream.Length}, original={originalSize}");
}
// Verify compression ratio
VerifyCompressionRatio(originalSize, zipStream.Length, expectedRatio, $"{compressionType} level {compressionLevel}");
// Verify archive content and CRC32
VerifyArchiveContent(zipStream, expectedFiles);
// Verify compression type is correctly set
VerifyCompressionType(zipStream, compressionType);
}
[Theory]
[InlineData(CompressionType.Deflate, 1, 4, 0.11f)] // was 0.8, actual 0.105
[InlineData(CompressionType.Deflate, 3, 4, 0.08f)] // was 0.8, actual 0.077
[InlineData(CompressionType.Deflate, 6, 4, 0.045f)] // was 0.8, actual 0.042
[InlineData(CompressionType.Deflate, 9, 4, 0.04f)] // was 0.8, actual 0.037
[InlineData(CompressionType.ZStandard, 1, 4, 0.025f)] // was 0.8, actual 0.022
[InlineData(CompressionType.ZStandard, 3, 4, 0.012f)] // was 0.8, actual 0.010
[InlineData(CompressionType.ZStandard, 9, 4, 0.003f)] // was 0.8, actual 0.002
[InlineData(CompressionType.ZStandard, 22, 4, 0.003f)] // was 0.8, actual 0.002
[InlineData(CompressionType.BZip2, 0, 4, 0.035f)] // was 0.8, actual 0.032
[InlineData(CompressionType.LZMA, 0, 4, 0.003f)] // was 0.8, actual 0.002
public void Zip_WriterFactory_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
{
var fileSize = sizeMb * 1024 * 1024;
var testData = TestPseudoTextStream.Create(fileSize);
var expectedCrc = CalculateCrc32(testData);
// Create archive with specified compression level
using var zipStream = new MemoryStream();
var writerOptions = new ZipWriterOptions(compressionType) { CompressionLevel = compressionLevel };
using (var writer = WriterFactory.Open(zipStream, ArchiveType.Zip, writerOptions))
{
writer.Write($"{compressionType}_level_{compressionLevel}_{sizeMb}MiB.txt", new MemoryStream(testData));
}
// Calculate and output actual compression ratio
var actualRatio = (double)zipStream.Length / testData.Length;
//Debug.WriteLine($"Zip_WriterFactory_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
VerifyCompressionRatio(testData.Length, zipStream.Length, expectedRatio, $"{compressionType} level {compressionLevel}");
// Verify the archive
zipStream.Position = 0;
using var archive = ZipArchive.Open(zipStream);
var entry = archive.Entries.Single(e => !e.IsDirectory);
using var entryStream = entry.OpenEntryStream();
using var extractedStream = new MemoryStream();
entryStream.CopyTo(extractedStream);
var extractedData = extractedStream.ToArray();
var actualCrc = CalculateCrc32(extractedData);
Assert.Equal(compressionType, entry.CompressionType);
Assert.Equal(expectedCrc, actualCrc);
Assert.Equal(testData.Length, extractedData.Length);
Assert.Equal(testData, extractedData);
}
[Theory]
[InlineData(CompressionType.Deflate, 1, 2, 0.11f)] // was 0.8, actual 0.104
[InlineData(CompressionType.Deflate, 3, 2, 0.08f)] // was 0.8, actual 0.077
[InlineData(CompressionType.Deflate, 6, 2, 0.045f)] // was 0.8, actual 0.042
[InlineData(CompressionType.Deflate, 9, 2, 0.04f)] // was 0.7, actual 0.038
[InlineData(CompressionType.ZStandard, 1, 2, 0.025f)] // was 0.8, actual 0.023
[InlineData(CompressionType.ZStandard, 3, 2, 0.015f)] // was 0.7, actual 0.012
[InlineData(CompressionType.ZStandard, 9, 2, 0.006f)] // was 0.7, actual 0.005
[InlineData(CompressionType.ZStandard, 22, 2, 0.005f)] // was 0.7, actual 0.004
[InlineData(CompressionType.BZip2, 0, 2, 0.035f)] // was 0.8, actual 0.032
[InlineData(CompressionType.LZMA, 0, 2, 0.005f)] // was 0.8, actual 0.004
public void Zip_ZipArchiveOpen_Crc32_Test(CompressionType compressionType, int compressionLevel, int sizeMb, float expectedRatio)
{
var fileSize = sizeMb * 1024 * 1024;
var testData = TestPseudoTextStream.Create(fileSize);
var expectedCrc = CalculateCrc32(testData);
// Create archive with specified compression and level
using var zipStream = new MemoryStream();
using (var writer = CreateWriterWithLevel(zipStream, compressionType, compressionLevel))
{
writer.Write($"{compressionType}_{compressionLevel}_{sizeMb}MiB.txt", new MemoryStream(testData));
}
// Calculate and output actual compression ratio
var actualRatio = (double)zipStream.Length / testData.Length;
//Debug.WriteLine($"Zip_ZipArchiveOpen_Crc32_Test: {compressionType} Level={compressionLevel} Size={sizeMb}MB Expected={expectedRatio:F3} Actual={actualRatio:F3}");
// Verify the archive
zipStream.Position = 0;
using var archive = ZipArchive.Open(zipStream);
var entry = archive.Entries.Single(e => !e.IsDirectory);
using var entryStream = entry.OpenEntryStream();
using var extractedStream = new MemoryStream();
entryStream.CopyTo(extractedStream);
var extractedData = extractedStream.ToArray();
var actualCrc = CalculateCrc32(extractedData);
Assert.Equal(compressionType, entry.CompressionType);
Assert.Equal(expectedCrc, actualCrc);
Assert.Equal(testData.Length, extractedData.Length);
// For smaller files, verify full content; for larger, spot check
if (testData.Length <= sizeMb * 2)
{
Assert.Equal(testData, extractedData);
}
else
{
VerifyDataSpotCheck(testData, extractedData);
}
VerifyCompressionRatio(testData.Length, zipStream.Length, expectedRatio, $"{compressionType} Level {compressionLevel}");
}
}