Start using buffers. Minimal in algorithms for now.

This commit is contained in:
Adam Hathcock
2016-10-03 11:09:09 +01:00
parent e5ee399045
commit 771986c50c
7 changed files with 355 additions and 218 deletions

View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using SharpCompress.Common;
using SharpCompress.Common.GZip;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.GZip;
using SharpCompress.Writers;
@@ -103,26 +104,28 @@ namespace SharpCompress.Archives.GZip
public static bool IsGZipFile(Stream stream)
{
// read the header on the first read
byte[] header = new byte[10];
int n = stream.Read(header, 0, header.Length);
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
using (var header = ByteArrayPool.RentScope(10))
{
return false;
}
int n = stream.Read(header);
if (n != 10)
{
return false;
}
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
{
return false;
}
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
return false;
}
if (n != 10)
{
return false;
}
return true;
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
return false;
}
return true;
}
}
/// <summary>

View File

@@ -5,6 +5,7 @@ using SharpCompress.Common.Tar.Headers;
using SharpCompress.Compressors;
using SharpCompress.Compressors.Deflate;
using SharpCompress.Converters;
using SharpCompress.IO;
namespace SharpCompress.Common.GZip
{
@@ -36,79 +37,85 @@ namespace SharpCompress.Common.GZip
private void ReadAndValidateGzipHeader(Stream stream)
{
// read the header on the first read
byte[] header = new byte[10];
int n = stream.Read(header, 0, header.Length);
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
using (var header = ByteArrayPool.RentScope(10))
{
return;
}
int n = stream.Read(header);
if (n != 10)
{
throw new ZlibException("Not a valid GZIP stream.");
}
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
throw new ZlibException("Bad GZIP header.");
}
Int32 timet = DataConverter.LittleEndian.GetInt32(header, 4);
DateModified = TarHeader.Epoch.AddSeconds(timet);
if ((header[3] & 0x04) == 0x04)
{
// read and discard extra field
n = stream.Read(header, 0, 2); // 2-byte length field
Int16 extraLength = (Int16)(header[0] + header[1] * 256);
byte[] extra = new byte[extraLength];
n = stream.Read(extra, 0, extra.Length);
if (n != extraLength)
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
{
throw new ZlibException("Unexpected end-of-file reading GZIP header.");
return;
}
if (n != 10)
{
throw new ZlibException("Not a valid GZIP stream.");
}
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
throw new ZlibException("Bad GZIP header.");
}
Int32 timet = DataConverter.LittleEndian.GetInt32(header.Array, 4);
DateModified = TarHeader.Epoch.AddSeconds(timet);
if ((header[3] & 0x04) == 0x04)
{
// read and discard extra field
n = stream.Read(header.Array, 0, 2); // 2-byte length field
Int16 extraLength = (Int16)(header[0] + header[1] * 256);
using (var extra = ByteArrayPool.RentScope(extraLength))
{
n = stream.Read(extra);
if (n != extraLength)
{
throw new ZlibException("Unexpected end-of-file reading GZIP header.");
}
}
}
if ((header[3] & 0x08) == 0x08)
{
name = ReadZeroTerminatedString(stream);
}
if ((header[3] & 0x10) == 0x010)
{
ReadZeroTerminatedString(stream);
}
if ((header[3] & 0x02) == 0x02)
{
stream.ReadByte(); // CRC16, ignore
}
}
if ((header[3] & 0x08) == 0x08)
{
name = ReadZeroTerminatedString(stream);
}
if ((header[3] & 0x10) == 0x010)
{
ReadZeroTerminatedString(stream);
}
if ((header[3] & 0x02) == 0x02)
{
stream.ReadByte(); // CRC16, ignore
}
}
private static string ReadZeroTerminatedString(Stream stream)
{
byte[] buf1 = new byte[1];
var list = new List<byte>();
bool done = false;
do
using (var buf1 = ByteArrayPool.RentScope(1))
{
// workitem 7740
int n = stream.Read(buf1, 0, 1);
if (n != 1)
var list = new List<byte>();
bool done = false;
do
{
throw new ZlibException("Unexpected EOF reading GZIP header.");
}
if (buf1[0] == 0)
{
done = true;
}
else
{
list.Add(buf1[0]);
// workitem 7740
int n = stream.Read(buf1);
if (n != 1)
{
throw new ZlibException("Unexpected EOF reading GZIP header.");
}
if (buf1[0] == 0)
{
done = true;
}
else
{
list.Add(buf1[0]);
}
}
while (!done);
byte[] a = list.ToArray();
return ArchiveEncoding.Default.GetString(a, 0, a.Length);
}
while (!done);
byte[] a = list.ToArray();
return ArchiveEncoding.Default.GetString(a, 0, a.Length);
}
}
}

View File

@@ -30,6 +30,7 @@ using System;
using System.IO;
using SharpCompress.Common;
using SharpCompress.Converters;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Deflate
{
@@ -413,67 +414,74 @@ namespace SharpCompress.Compressors.Deflate
int fnLength = (FileName == null) ? 0 : filenameBytes.Length + 1;
int bufferLength = 10 + cbLength + fnLength;
var header = new byte[bufferLength];
int i = 0;
// ID
header[i++] = 0x1F;
header[i++] = 0x8B;
// compression method
header[i++] = 8;
byte flag = 0;
if (Comment != null)
var header = ByteArrayPool.RentWritable(bufferLength);
try
{
flag ^= 0x10;
int i = 0;
// ID
header[i++] = 0x1F;
header[i++] = 0x8B;
// compression method
header[i++] = 8;
byte flag = 0;
if (Comment != null)
{
flag ^= 0x10;
}
if (FileName != null)
{
flag ^= 0x8;
}
// flag
header[i++] = flag;
// mtime
if (!LastModified.HasValue)
{
LastModified = DateTime.Now;
}
TimeSpan delta = LastModified.Value - UnixEpoch;
var timet = (Int32)delta.TotalSeconds;
DataConverter.LittleEndian.PutBytes(header, i, timet);
i += 4;
// xflg
header[i++] = 0; // this field is totally useless
// OS
header[i++] = 0xFF; // 0xFF == unspecified
// extra field length - only if FEXTRA is set, which it is not.
//header[i++]= 0;
//header[i++]= 0;
// filename
if (fnLength != 0)
{
Array.Copy(filenameBytes, 0, header, i, fnLength - 1);
i += fnLength - 1;
header[i++] = 0; // terminate
}
// comment
if (cbLength != 0)
{
Array.Copy(commentBytes, 0, header, i, cbLength - 1);
i += cbLength - 1;
header[i++] = 0; // terminate
}
BaseStream._stream.Write(header, 0, header.Length);
return header.Length; // bytes written
}
if (FileName != null)
finally
{
flag ^= 0x8;
ByteArrayPool.Return(header);
}
// flag
header[i++] = flag;
// mtime
if (!LastModified.HasValue)
{
LastModified = DateTime.Now;
}
TimeSpan delta = LastModified.Value - UnixEpoch;
var timet = (Int32)delta.TotalSeconds;
DataConverter.LittleEndian.PutBytes(header, i, timet);
i += 4;
// xflg
header[i++] = 0; // this field is totally useless
// OS
header[i++] = 0xFF; // 0xFF == unspecified
// extra field length - only if FEXTRA is set, which it is not.
//header[i++]= 0;
//header[i++]= 0;
// filename
if (fnLength != 0)
{
Array.Copy(filenameBytes, 0, header, i, fnLength - 1);
i += fnLength - 1;
header[i++] = 0; // terminate
}
// comment
if (cbLength != 0)
{
Array.Copy(commentBytes, 0, header, i, cbLength - 1);
i += cbLength - 1;
header[i++] = 0; // terminate
}
BaseStream._stream.Write(header, 0, header.Length);
return header.Length; // bytes written
}
}
}

View File

@@ -30,6 +30,7 @@ using System.IO;
using SharpCompress.Common;
using SharpCompress.Common.Tar.Headers;
using SharpCompress.Converters;
using SharpCompress.IO;
namespace SharpCompress.Compressors.Deflate
{
@@ -267,46 +268,48 @@ namespace SharpCompress.Compressors.Deflate
}
// Read and potentially verify the GZIP trailer: CRC32 and size mod 2^32
byte[] trailer = new byte[8];
// workitem 8679
if (_z.AvailableBytesIn != 8)
using (var trailer = ByteArrayPool.RentScope(8))
{
// Make sure we have read to the end of the stream
Array.Copy(_z.InputBuffer, _z.NextIn, trailer, 0, _z.AvailableBytesIn);
int bytesNeeded = 8 - _z.AvailableBytesIn;
int bytesRead = _stream.Read(trailer,
_z.AvailableBytesIn,
bytesNeeded);
if (bytesNeeded != bytesRead)
// workitem 8679
if (_z.AvailableBytesIn != 8)
{
throw new ZlibException(String.Format(
"Protocol error. AvailableBytesIn={0}, expected 8",
_z.AvailableBytesIn + bytesRead));
// Make sure we have read to the end of the stream
Array.Copy(_z.InputBuffer, _z.NextIn, trailer.Array, 0, _z.AvailableBytesIn);
int bytesNeeded = 8 - _z.AvailableBytesIn;
int bytesRead = _stream.Read(trailer.Array,
_z.AvailableBytesIn,
bytesNeeded);
if (bytesNeeded != bytesRead)
{
throw new ZlibException(String.Format(
"Protocol error. AvailableBytesIn={0}, expected 8",
_z.AvailableBytesIn + bytesRead));
}
}
else
{
Array.Copy(_z.InputBuffer, _z.NextIn, trailer.Array, 0, trailer.Count);
}
}
else
{
Array.Copy(_z.InputBuffer, _z.NextIn, trailer, 0, trailer.Length);
}
Int32 crc32_expected = DataConverter.LittleEndian.GetInt32(trailer, 0);
Int32 crc32_actual = crc.Crc32Result;
Int32 isize_expected = DataConverter.LittleEndian.GetInt32(trailer, 4);
Int32 isize_actual = (Int32)(_z.TotalBytesOut & 0x00000000FFFFFFFF);
Int32 crc32_expected = DataConverter.LittleEndian.GetInt32(trailer.Array, 0);
Int32 crc32_actual = crc.Crc32Result;
Int32 isize_expected = DataConverter.LittleEndian.GetInt32(trailer.Array, 4);
Int32 isize_actual = (Int32)(_z.TotalBytesOut & 0x00000000FFFFFFFF);
if (crc32_actual != crc32_expected)
{
throw new ZlibException(
String.Format("Bad CRC32 in GZIP stream. (actual({0:X8})!=expected({1:X8}))",
crc32_actual, crc32_expected));
}
if (crc32_actual != crc32_expected)
{
throw new ZlibException(
String.Format("Bad CRC32 in GZIP stream. (actual({0:X8})!=expected({1:X8}))",
crc32_actual, crc32_expected));
}
if (isize_actual != isize_expected)
{
throw new ZlibException(
String.Format("Bad size in GZIP stream. (actual({0})!=expected({1}))", isize_actual,
isize_expected));
if (isize_actual != isize_expected)
{
throw new ZlibException(
String.Format("Bad size in GZIP stream. (actual({0})!=expected({1}))", isize_actual,
isize_expected));
}
}
}
else
@@ -427,57 +430,61 @@ namespace SharpCompress.Compressors.Deflate
int totalBytesRead = 0;
// read the header on the first read
byte[] header = new byte[10];
int n = _stream.Read(header, 0, header.Length);
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
using (var header = ByteArrayPool.RentScope(10))
{
return 0;
}
int n = _stream.Read(header);
if (n != 10)
{
throw new ZlibException("Not a valid GZIP stream.");
}
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
throw new ZlibException("Bad GZIP header.");
}
Int32 timet = DataConverter.LittleEndian.GetInt32(header, 4);
_GzipMtime = TarHeader.Epoch.AddSeconds(timet);
totalBytesRead += n;
if ((header[3] & 0x04) == 0x04)
{
// read and discard extra field
n = _stream.Read(header, 0, 2); // 2-byte length field
totalBytesRead += n;
Int16 extraLength = (Int16)(header[0] + header[1] * 256);
byte[] extra = new byte[extraLength];
n = _stream.Read(extra, 0, extra.Length);
if (n != extraLength)
// workitem 8501: handle edge case (decompress empty stream)
if (n == 0)
{
throw new ZlibException("Unexpected end-of-file reading GZIP header.");
return 0;
}
totalBytesRead += n;
}
if ((header[3] & 0x08) == 0x08)
{
_GzipFileName = ReadZeroTerminatedString();
}
if ((header[3] & 0x10) == 0x010)
{
_GzipComment = ReadZeroTerminatedString();
}
if ((header[3] & 0x02) == 0x02)
{
Read(_buf1, 0, 1); // CRC16, ignore
}
return totalBytesRead;
if (n != 10)
{
throw new ZlibException("Not a valid GZIP stream.");
}
if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8)
{
throw new ZlibException("Bad GZIP header.");
}
Int32 timet = DataConverter.LittleEndian.GetInt32(header.Array, 4);
_GzipMtime = TarHeader.Epoch.AddSeconds(timet);
totalBytesRead += n;
if ((header[3] & 0x04) == 0x04)
{
// read and discard extra field
n = _stream.Read(header.Array, 0, 2); // 2-byte length field
totalBytesRead += n;
Int16 extraLength = (Int16)(header[0] + header[1] * 256);
using (var extra = ByteArrayPool.RentScope(extraLength))
{
n = _stream.Read(extra);
if (n != extraLength)
{
throw new ZlibException("Unexpected end-of-file reading GZIP header.");
}
totalBytesRead += n;
}
}
if ((header[3] & 0x08) == 0x08)
{
_GzipFileName = ReadZeroTerminatedString();
}
if ((header[3] & 0x10) == 0x010)
{
_GzipComment = ReadZeroTerminatedString();
}
if ((header[3] & 0x02) == 0x02)
{
Read(_buf1, 0, 1); // CRC16, ignore
}
return totalBytesRead;
}
}
public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count)

View File

@@ -0,0 +1,52 @@
using System;
#if !NO_BUFFERS
using System.Buffers;
#endif
namespace SharpCompress.IO
{
public static class ByteArrayPool
{
#if !NO_BUFFERS
private static readonly ArrayPool<byte> POOL = ArrayPool<byte>.Create();
#endif
public static ArraySegment<byte> Rent(int size)
{
#if NO_BUFFERS
return new ArraySegment<byte>(new byte[size]);
#else
return new ArraySegment<byte>(POOL.Rent(size), 0, size);
#endif
}
public static byte[] RentWritable(int size)
{
#if NO_BUFFERS
return new byte[size];
#else
return POOL.Rent(size);
#endif
}
public static ByteArrayPoolScope RentScope(int size)
{
return new ByteArrayPoolScope(Rent(size));
}
public static void Return(ArraySegment<byte> array)
{
#if NO_BUFFERS
#else
POOL.Return(array.Array);
#endif
}
public static void Return(byte[] array)
{
#if NO_BUFFERS
#else
POOL.Return(array);
#endif
}
}
}

View File

@@ -0,0 +1,37 @@
using System;
using System.IO;
namespace SharpCompress.IO
{
public struct ByteArrayPoolScope : IDisposable
{
private readonly ArraySegment<byte> array;
public ByteArrayPoolScope(ArraySegment<byte> array)
{
this.array = array;
}
public byte[] Array => array.Array;
public int Offset => array.Offset;
public int Count => array.Count;
public byte this[int index]
{
get { return Array[index]; }
set { Array[index] = value; }
}
public void Dispose()
{
ByteArrayPool.Return(array);
}
}
public static class ByteArrayPoolScopeExtensions
{
public static int Read(this Stream stream, ByteArrayPoolScope scope)
{
return stream.Read(scope.Array, scope.Offset, scope.Count);
}
}
}

View File

@@ -16,15 +16,22 @@
"allowUnsafe": true
},
"frameworks": {
"net35": {
},
"net40": {
"buildOptions": {
"define": [ "NO_BUFFERS" ]
}
},
"net45": {
"dependencies": {
"System.Buffers": "4.0.0"
},
"frameworkAssemblies": {
"System.Runtime": { "type": "build" }
}
},
".NETPortable,Version=v4.0,Profile=Profile328": {
"buildOptions": {
"define": [ "NO_FILE", "NO_CRYPTO", "SILVERLIGHT" ]
"define": [ "NO_FILE", "NO_CRYPTO", "SILVERLIGHT", "NO_BUFFERS" ]
},
"frameworkAssemblies": {
"mscorlib": { "type": "build" },
@@ -34,7 +41,7 @@
},
".NETPortable,Version=v4.5,Profile=Profile259": {
"buildOptions": {
"define": [ "NO_FILE", "NO_CRYPTO", "SILVERLIGHT" ]
"define": [ "NO_FILE", "NO_CRYPTO", "SILVERLIGHT", "NO_BUFFERS" ]
},
"frameworkAssemblies": {
"System": { "type": "build" },
@@ -52,7 +59,7 @@
},
"netstandard1.0": {
"buildOptions": {
"define": [ "NO_FILE", "NO_CRYPTO" ]
"define": [ "NO_FILE", "NO_CRYPTO", "NO_BUFFERS" ]
},
"dependencies": {
"System.Collections": "4.0.11",
@@ -64,8 +71,24 @@
"System.Text.Encoding.Extensions": "4.0.11"
}
},
"netstandard1.1": {
"buildOptions": {
"define": [ "NO_FILE", "NO_CRYPTO" ]
},
"dependencies": {
"System.Buffers": "4.0.0",
"System.Collections": "4.0.11",
"System.Diagnostics.Debug": "4.0.11",
"System.IO": "4.1.0",
"System.Linq": "4.1.0",
"System.Resources.ResourceManager": "4.0.1",
"System.Runtime.Extensions": "4.1.0",
"System.Text.Encoding.Extensions": "4.0.11"
}
},
"netstandard1.3": {
"dependencies": {
"System.Buffers": "4.0.0",
"System.Collections": "4.0.11",
"System.Diagnostics.Debug": "4.0.11",
"System.IO": "4.1.0",