claunia/RomVault
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Thanks to test work done by EDC, this is based on that code, added double buffered file reading, and multi-threader hashing in zip and file checking. V2.2.5

Browse Source
This commit is contained in:
gjefferyes
2016-02-12 16:02:48 -06:00
parent 905574d120
commit 5a93500001
5 changed files with 104 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ROMVault2/DatReaders/DatMessXMLReader.cs
View File

@@ -16,7 +16,7 @@ namespace ROMVault2.DatReaders
public static bool ReadDat(ref RvDir tDat, XmlDocument doc) public static bool ReadDat(ref RvDir tDat, XmlDocument doc)
{ {
FileType thisFileType = FileType.Unknown; // added FileType thisFileType = FileType.Unknown;
if (!LoadHeaderFromDat(ref tDat, ref doc, ref thisFileType)) if (!LoadHeaderFromDat(ref tDat, ref doc, ref thisFileType))
return false; return false;
@@ -65,7 +65,6 @@ namespace ROMVault2.DatReaders
break; break;
} }
// deterime whether to compress to .zip or store as files
val = VarFix.String(head[0].Attributes.GetNamedItem("forcepacking")).ToLower(); val = VarFix.String(head[0].Attributes.GetNamedItem("forcepacking")).ToLower();
switch (val.ToLower()) switch (val.ToLower())
{ {

2
ROMVault2/Program.cs
View File

@@ -15,7 +15,7 @@ namespace ROMVault2
//public static UsernamePassword Up; //public static UsernamePassword Up;
public static readonly Encoding Enc = Encoding.GetEncoding(28591); public static readonly Encoding Enc = Encoding.GetEncoding(28591);
public const string Version = "2.2"; public const string Version = "2.2";
public const int SubVersion = 4; public const int SubVersion = 5;
public static SynchronizationContext SyncCont; public static SynchronizationContext SyncCont;

2
ROMVault2/ROMVault2.csproj
View File

@@ -9,7 +9,7 @@
<OutputType>WinExe</OutputType> <OutputType>WinExe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>ROMVault2</RootNamespace> <RootNamespace>ROMVault2</RootNamespace>
<AssemblyName>ROMVault22</AssemblyName> <AssemblyName>ROMVault225</AssemblyName>
<TargetFrameworkVersion>v3.5</TargetFrameworkVersion> <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
<TargetFrameworkProfile> <TargetFrameworkProfile>
</TargetFrameworkProfile> </TargetFrameworkProfile>

65
ROMVault2/SupportedFiles/Files/UnCompFiles.cs
View File

@@ -6,18 +6,21 @@
using System.IO; using System.IO;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Threading;
using ROMVault2.SupportedFiles.Zip.ZLib; using ROMVault2.SupportedFiles.Zip.ZLib;
namespace ROMVault2.SupportedFiles.Files namespace ROMVault2.SupportedFiles.Files
{ {
public static class UnCompFiles public static class UnCompFiles
{ {
private const int Buffersize = 4096; private const int Buffersize = 4096 * 1024;
private static readonly byte[] Buffer; private static readonly byte[] Buffer0;
private static readonly byte[] Buffer1;
static UnCompFiles() static UnCompFiles()
{ {
Buffer = new byte[Buffersize]; Buffer0 = new byte[Buffersize];
Buffer1 = new byte[Buffersize];
} }
public static int CheckSumRead(string filename, bool testDeep, out byte[] crc, out byte[] bMD5, out byte[] bSHA1) public static int CheckSumRead(string filename, bool testDeep, out byte[] crc, out byte[] bMD5, out byte[] bSHA1)
@@ -26,7 +29,7 @@ namespace ROMVault2.SupportedFiles.Files
bSHA1 = null; bSHA1 = null;
crc = null; crc = null;
Stream ds=null; Stream ds = null;
CRC32Hash crc32 = new CRC32Hash(); CRC32Hash crc32 = new CRC32Hash();
MD5 md5 = null; MD5 md5 = null;
@@ -42,20 +45,48 @@ namespace ROMVault2.SupportedFiles.Files
long sizetogo = ds.Length; long sizetogo = ds.Length;
while (sizetogo > 0) // Pre load the first buffer0
{ int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
int sizenow = sizetogo > Buffersize ? Buffersize : (int)sizetogo; ds.Read(Buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= sizeNext;
bool whichBuffer = true;
ds.Read(Buffer, 0, sizenow); while (sizebuffer > 0)
crc32.TransformBlock(Buffer, 0, sizenow, null, 0); {
if (testDeep) md5.TransformBlock(Buffer, 0, sizenow, null, 0); sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (testDeep) sha1.TransformBlock(Buffer, 0, sizenow, null, 0);
sizetogo -= sizenow; Thread t0 = null;
if (sizeNext > 0)
{
t0 = new Thread(() => { ds.Read(whichBuffer ? Buffer1 : Buffer0, 0, sizeNext); });
t0.Start();
}
byte[] buffer = whichBuffer ? Buffer0 : Buffer1;
Thread t1 = new Thread(() => { crc32.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t1.Start();
if (testDeep)
{
Thread t2 = new Thread(() => { md5.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t3 = new Thread(() => { sha1.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t2.Start();
t3.Start();
t2.Join();
t3.Join();
}
if (t0 != null)
t0.Join();
t1.Join();
sizebuffer = sizeNext;
sizetogo -= sizeNext;
whichBuffer = !whichBuffer;
} }
crc32.TransformFinalBlock(Buffer, 0, 0); crc32.TransformFinalBlock(Buffer0, 0, 0);
if (testDeep) md5.TransformFinalBlock(Buffer, 0, 0); if (testDeep) md5.TransformFinalBlock(Buffer0, 0, 0);
if (testDeep) sha1.TransformFinalBlock(Buffer, 0, 0); if (testDeep) sha1.TransformFinalBlock(Buffer0, 0, 0);
ds.Close(); ds.Close();
} }
@@ -67,15 +98,11 @@ namespace ROMVault2.SupportedFiles.Files
return 0x17; return 0x17;
} }
crc = crc32.Hash; crc = crc32.Hash;
if (testDeep) bMD5 = md5.Hash; if (testDeep) bMD5 = md5.Hash;
if (testDeep) bSHA1 = sha1.Hash; if (testDeep) bSHA1 = sha1.Hash;
return 0; return 0;
} }
} }
} }

80
ROMVault2/SupportedFiles/Zip/zipFile.cs
View File

@@ -9,6 +9,7 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading;
using ROMVault2.SupportedFiles.Zip.ZLib; using ROMVault2.SupportedFiles.Zip.ZLib;
// UInt16 = ushort // UInt16 = ushort
@@ -17,11 +18,12 @@ using ROMVault2.SupportedFiles.Zip.ZLib;
namespace ROMVault2.SupportedFiles.Zip namespace ROMVault2.SupportedFiles.Zip
{ {
public class ZipFile public class ZipFile
{ {
const int Buffersize = 4096 * 128; const int Buffersize = 4096 * 1024;
private static byte[] _buffer; private static byte[] _buffer0;
private static byte[] _buffer1;
private const uint LocalFileHeaderSignature = 0x04034b50; private const uint LocalFileHeaderSignature = 0x04034b50;
private const uint CentralDirectoryHeaderSigniature = 0x02014b50; private const uint CentralDirectoryHeaderSigniature = 0x02014b50;
@@ -48,10 +50,10 @@ namespace ROMVault2.SupportedFiles.Zip
public bool Zip64 { get; private set; } public bool Zip64 { get; private set; }
public bool TrrntZip { get; private set; } public bool TrrntZip { get; private set; }
public byte[] sha1 { get; private set; } public byte[] sha1 { get; private set; }
public byte[] md5 { get; private set; } public byte[] md5 { get; private set; }
public ZipReturn FileStatus = ZipReturn.ZipUntested; public ZipReturn FileStatus = ZipReturn.ZipUntested;
public LocalFile(Stream zipFs) public LocalFile(Stream zipFs)
{ {
@@ -414,7 +416,7 @@ namespace ROMVault2.SupportedFiles.Zip
_generalPurposeBitFlag = br.ReadUInt16(); _generalPurposeBitFlag = br.ReadUInt16();
if ((_generalPurposeBitFlag & 8) == 8) if ((_generalPurposeBitFlag & 8) == 8)
return ZipReturn.ZipCannotFastOpen; return ZipReturn.ZipCannotFastOpen;
_compressionMethod = br.ReadUInt16(); _compressionMethod = br.ReadUInt16();
_lastModFileTime = br.ReadUInt16(); _lastModFileTime = br.ReadUInt16();
_lastModFileDate = br.ReadUInt16(); _lastModFileDate = br.ReadUInt16();
@@ -482,7 +484,7 @@ namespace ROMVault2.SupportedFiles.Zip
_dataLocation = (ulong)_zipFs.Position; _dataLocation = (ulong)_zipFs.Position;
return ZipReturn.ZipGood; return ZipReturn.ZipGood;
} }
catch catch
{ {
@@ -717,25 +719,53 @@ namespace ROMVault2.SupportedFiles.Zip
MD5 lmd5 = System.Security.Cryptography.MD5.Create(); MD5 lmd5 = System.Security.Cryptography.MD5.Create();
SHA1 lsha1 = System.Security.Cryptography.SHA1.Create(); SHA1 lsha1 = System.Security.Cryptography.SHA1.Create();
ulong sizetogo = UncompressedSize; if (_buffer0 == null)
if (_buffer == null)
_buffer = new byte[Buffersize];
while (sizetogo > 0)
{ {
int sizenow = sizetogo > Buffersize ? Buffersize : (int)sizetogo; _buffer0 = new byte[Buffersize];
sInput.Read(_buffer, 0, sizenow); _buffer1 = new byte[Buffersize];
crc32.TransformBlock(_buffer, 0, sizenow, null, 0);
lmd5.TransformBlock(_buffer, 0, sizenow, null, 0);
lsha1.TransformBlock(_buffer, 0, sizenow, null, 0);
sizetogo = sizetogo - (ulong)sizenow;
} }
crc32.TransformFinalBlock(_buffer, 0, 0); ulong sizetogo = UncompressedSize;
lmd5.TransformFinalBlock(_buffer, 0, 0);
lsha1.TransformFinalBlock(_buffer, 0, 0); // Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
sInput.Read(_buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
bool whichBuffer = true;
while (sizebuffer > 0)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
Thread t0 = null;
if (sizeNext > 0)
{
t0 = new Thread(() => { sInput.Read(whichBuffer ? _buffer1 : _buffer0, 0, sizeNext); });
t0.Start();
}
byte[] buffer = whichBuffer ? _buffer0 : _buffer1;
Thread t1 = new Thread(() => { crc32.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t2 = new Thread(() => { lmd5.TransformBlock(buffer, 0, sizebuffer, null, 0); });
Thread t3 = new Thread(() => { lsha1.TransformBlock(buffer, 0, sizebuffer, null, 0); });
t1.Start();
t2.Start();
t3.Start();
if (t0 != null)
t0.Join();
t1.Join();
t2.Join();
t3.Join();
sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
whichBuffer = !whichBuffer;
}
crc32.TransformFinalBlock(_buffer0, 0, 0);
lmd5.TransformFinalBlock(_buffer0, 0, 0);
lsha1.TransformFinalBlock(_buffer0, 0, 0);
byte[] testcrc = crc32.Hash; byte[] testcrc = crc32.Hash;
md5 = lmd5.Hash; md5 = lmd5.Hash;
@@ -1365,7 +1395,7 @@ namespace ROMVault2.SupportedFiles.Zip
_localFiles[_localFiles.Count - 1].LocalFileAddDirectory(); _localFiles[_localFiles.Count - 1].LocalFileAddDirectory();
} }
/* /*
public void BreakTrrntZip(string filename) public void BreakTrrntZip(string filename)
{ {
@@ -1525,7 +1555,7 @@ namespace ROMVault2.SupportedFiles.Zip
int pos1 = 0; int pos1 = 0;
int pos2 = 0; int pos2 = 0;
for (; ; ) for (;;)
{ {
if (pos1 == bytes1.Length) if (pos1 == bytes1.Length)
return ((pos2 == bytes2.Length) ? 0 : -1); return ((pos2 == bytes2.Length) ? 0 : -1);