claunia/RomVault
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Thanks to test work done by EDC, this is based on that code, added double buffered file reading, and multi-threader hashing in zip and file checking. V2.2.5

Browse Source
This commit is contained in:
gjefferyes
2016-02-12 16:02:48 -06:00
parent 905574d120
commit 5a93500001
5 changed files with 104 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ROMVault2/DatReaders/DatMessXMLReader.cs
View File

@@ -16,7 +16,7 @@ namespace ROMVault2.DatReaders
public static bool ReadDat(ref RvDir tDat, XmlDocument doc)
{
FileType thisFileType = FileType.Unknown; // added
FileType thisFileType = FileType.Unknown;
if (!LoadHeaderFromDat(ref tDat, ref doc, ref thisFileType))
return false;
@@ -65,7 +65,6 @@ namespace ROMVault2.DatReaders
break;
}
// deterime whether to compress to .zip or store as files
val = VarFix.String(head[0].Attributes.GetNamedItem("forcepacking")).ToLower();
switch (val.ToLower())
{

2
ROMVault2/Program.cs
View File

@@ -15,7 +15,7 @@ namespace ROMVault2
//public static UsernamePassword Up;
public static readonly Encoding Enc = Encoding.GetEncoding(28591);
public const string Version = "2.2";
public const int SubVersion = 4;
public const int SubVersion = 5;
public static SynchronizationContext SyncCont;

2
ROMVault2/ROMVault2.csproj
View File

@@ -9,7 +9,7 @@
<OutputType>WinExe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>ROMVault2</RootNamespace>
<AssemblyName>ROMVault22</AssemblyName>
<AssemblyName>ROMVault225</AssemblyName>
<TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
<TargetFrameworkProfile>
</TargetFrameworkProfile>

65
ROMVault2/SupportedFiles/Files/UnCompFiles.cs
View File

@@ -6,18 +6,21 @@
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using ROMVault2.SupportedFiles.Zip.ZLib;
namespace ROMVault2.SupportedFiles.Files
{
public static class UnCompFiles
{
private const int Buffersize = 4096;
private static readonly byte[] Buffer;
private const int Buffersize = 4096 * 1024;
private static readonly byte[] Buffer0;
private static readonly byte[] Buffer1;
static UnCompFiles()
{
Buffer = new byte[Buffersize];
Buffer0 = new byte[Buffersize];
Buffer1 = new byte[Buffersize];
}
public static int CheckSumRead(string filename, bool testDeep, out byte[] crc, out byte[] bMD5, out byte[] bSHA1)
@@ -26,7 +29,7 @@ namespace ROMVault2.SupportedFiles.Files
bSHA1 = null;
crc = null;
Stream ds=null;
Stream ds = null;
CRC32Hash crc32 = new CRC32Hash();
MD5 md5 = null;
@@ -42,20 +45,48 @@ namespace ROMVault2.SupportedFiles.Files
long sizetogo = ds.Length;
while (sizetogo > 0)
{
int sizenow = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
ds.Read(Buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= sizeNext;
bool whichBuffer = true;
ds.Read(Buffer, 0, sizenow);
crc32.TransformBlock(Buffer, 0, sizenow, null, 0);
if (testDeep) md5.TransformBlock(Buffer, 0, sizenow, null, 0);
if (testDeep) sha1.TransformBlock(Buffer, 0, sizenow, null, 0);
sizetogo -= sizenow;
while (sizebuffer > 0)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
Thread t0 = null;
if (sizeNext > 0)
{
t0 = new Thread(() => { ds.Read(whichBuffer ? Buffer1 : Buffer0, 0, sizeNext); });
t0.Start();
}
byte[] buffer = whichBuffer ? Buffer0 : Buffer1;
Thread t1 = new Thread(() => { crc32.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t1.Start();
if (testDeep)
{
Thread t2 = new Thread(() => { md5.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t3 = new Thread(() => { sha1.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t2.Start();
t3.Start();
t2.Join();
t3.Join();
}
if (t0 != null)
t0.Join();
t1.Join();
sizebuffer = sizeNext;
sizetogo -= sizeNext;
whichBuffer = !whichBuffer;
}
crc32.TransformFinalBlock(Buffer, 0, 0);
if (testDeep) md5.TransformFinalBlock(Buffer, 0, 0);
if (testDeep) sha1.TransformFinalBlock(Buffer, 0, 0);
crc32.TransformFinalBlock(Buffer0, 0, 0);
if (testDeep) md5.TransformFinalBlock(Buffer0, 0, 0);
if (testDeep) sha1.TransformFinalBlock(Buffer0, 0, 0);
ds.Close();
}
@@ -67,15 +98,11 @@ namespace ROMVault2.SupportedFiles.Files
return 0x17;
}
crc = crc32.Hash;
if (testDeep) bMD5 = md5.Hash;
if (testDeep) bSHA1 = sha1.Hash;
return 0;
}
}
}

80
ROMVault2/SupportedFiles/Zip/zipFile.cs
View File

@@ -9,6 +9,7 @@ using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using ROMVault2.SupportedFiles.Zip.ZLib;
// UInt16 = ushort
@@ -17,11 +18,12 @@ using ROMVault2.SupportedFiles.Zip.ZLib;
namespace ROMVault2.SupportedFiles.Zip
{
public class ZipFile
{
const int Buffersize = 4096 * 128;
private static byte[] _buffer;
const int Buffersize = 4096 * 1024;
private static byte[] _buffer0;
private static byte[] _buffer1;
private const uint LocalFileHeaderSignature = 0x04034b50;
private const uint CentralDirectoryHeaderSigniature = 0x02014b50;
@@ -48,10 +50,10 @@ namespace ROMVault2.SupportedFiles.Zip
public bool Zip64 { get; private set; }
public bool TrrntZip { get; private set; }
public byte[] sha1 { get; private set; }
public byte[] md5 { get; private set; }
public ZipReturn FileStatus = ZipReturn.ZipUntested;
public LocalFile(Stream zipFs)
{
@@ -414,7 +416,7 @@ namespace ROMVault2.SupportedFiles.Zip
_generalPurposeBitFlag = br.ReadUInt16();
if ((_generalPurposeBitFlag & 8) == 8)
return ZipReturn.ZipCannotFastOpen;
_compressionMethod = br.ReadUInt16();
_lastModFileTime = br.ReadUInt16();
_lastModFileDate = br.ReadUInt16();
@@ -482,7 +484,7 @@ namespace ROMVault2.SupportedFiles.Zip
_dataLocation = (ulong)_zipFs.Position;
return ZipReturn.ZipGood;
}
catch
{
@@ -717,25 +719,53 @@ namespace ROMVault2.SupportedFiles.Zip
MD5 lmd5 = System.Security.Cryptography.MD5.Create();
SHA1 lsha1 = System.Security.Cryptography.SHA1.Create();
ulong sizetogo = UncompressedSize;
if (_buffer == null)
_buffer = new byte[Buffersize];
while (sizetogo > 0)
if (_buffer0 == null)
{
int sizenow = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
sInput.Read(_buffer, 0, sizenow);
crc32.TransformBlock(_buffer, 0, sizenow, null, 0);
lmd5.TransformBlock(_buffer, 0, sizenow, null, 0);
lsha1.TransformBlock(_buffer, 0, sizenow, null, 0);
sizetogo = sizetogo - (ulong)sizenow;
_buffer0 = new byte[Buffersize];
_buffer1 = new byte[Buffersize];
}
crc32.TransformFinalBlock(_buffer, 0, 0);
lmd5.TransformFinalBlock(_buffer, 0, 0);
lsha1.TransformFinalBlock(_buffer, 0, 0);
ulong sizetogo = UncompressedSize;
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
sInput.Read(_buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
bool whichBuffer = true;
while (sizebuffer > 0)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
Thread t0 = null;
if (sizeNext > 0)
{
t0 = new Thread(() => { sInput.Read(whichBuffer ? _buffer1 : _buffer0, 0, sizeNext); });
t0.Start();
}
byte[] buffer = whichBuffer ? _buffer0 : _buffer1;
Thread t1 = new Thread(() => { crc32.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t2 = new Thread(() => { lmd5.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t3 = new Thread(() => { lsha1.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t1.Start();
t2.Start();
t3.Start();
if (t0 != null)
t0.Join();
t1.Join();
t2.Join();
t3.Join();
sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
whichBuffer = !whichBuffer;
}
crc32.TransformFinalBlock(_buffer0, 0, 0);
lmd5.TransformFinalBlock(_buffer0, 0, 0);
lsha1.TransformFinalBlock(_buffer0, 0, 0);
byte[] testcrc = crc32.Hash;
md5 = lmd5.Hash;
@@ -1365,7 +1395,7 @@ namespace ROMVault2.SupportedFiles.Zip
_localFiles[_localFiles.Count - 1].LocalFileAddDirectory();
}
/*
public void BreakTrrntZip(string filename)
{
@@ -1525,7 +1555,7 @@ namespace ROMVault2.SupportedFiles.Zip
int pos1 = 0;
int pos2 = 0;
for (; ; )
for (;;)
{
if (pos1 == bytes1.Length)
return ((pos2 == bytes2.Length) ? 0 : -1);