diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index 34989b49..e794359a 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; +using SabreTools.Data; using SabreTools.Help; using SabreTools.IO; using SabreTools.Library.DatFiles; diff --git a/SabreTools.Data/Enums.cs b/SabreTools.Data/Enums.cs new file mode 100644 index 00000000..c12bf9de --- /dev/null +++ b/SabreTools.Data/Enums.cs @@ -0,0 +1,32 @@ +using System; + +namespace SabreTools.Data +{ + /// + /// Available hashing types + /// + [Flags] + public enum Hash + { + CRC = 1 << 0, + MD5 = 1 << 1, +#if NET_FRAMEWORK + RIPEMD160 = 1 << 2, +#endif + SHA1 = 1 << 3, + SHA256 = 1 << 4, + SHA384 = 1 << 5, + SHA512 = 1 << 6, + SpamSum = 1 << 7, + + // Special combinations + Standard = CRC | MD5 | SHA1, +#if NET_FRAMEWORK + DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum, + SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, +#else + DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum, + SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, +#endif + } +} diff --git a/SabreTools.Library/External/Aaru/IChecksum.cs b/SabreTools.IO/Aaru/IChecksum.cs similarity index 100% rename from SabreTools.Library/External/Aaru/IChecksum.cs rename to SabreTools.IO/Aaru/IChecksum.cs diff --git a/SabreTools.Library/External/Aaru/SpamSumContext.cs b/SabreTools.IO/Aaru/SpamSumContext.cs similarity index 99% rename from SabreTools.Library/External/Aaru/SpamSumContext.cs rename to SabreTools.IO/Aaru/SpamSumContext.cs index cd4eb26a..2810fc9c 100644 --- a/SabreTools.Library/External/Aaru/SpamSumContext.cs +++ b/SabreTools.IO/Aaru/SpamSumContext.cs @@ -506,7 +506,13 @@ namespace Aaru.Checksums count++; } +#if NET_FRAMEWORK + byte[] newString = new byte[count]; + Array.Copy(cString, newString, count); + return newString; +#else return new ReadOnlySpan(cString, 0, count).ToArray(); +#endif } public void Dispose() diff --git a/SabreTools.Library/Tools/Hasher.cs b/SabreTools.IO/Hasher.cs similarity index 91% rename from SabreTools.Library/Tools/Hasher.cs rename to SabreTools.IO/Hasher.cs index 0133ec2a..1a443288 100644 --- a/SabreTools.Library/Tools/Hasher.cs +++ b/SabreTools.IO/Hasher.cs @@ -3,10 +3,9 @@ using System.Linq; using System.Security.Cryptography; using Aaru.Checksums; -using SabreTools.Library.DatFiles; -using SabreTools.Library.External; +using SabreTools.Data; -namespace SabreTools.Library.Tools +namespace SabreTools.IO { /// /// Async hashing class wraper @@ -30,7 +29,7 @@ namespace SabreTools.Library.Tools switch (HashType) { case Hash.CRC: - _hasher = new OptimizedCRC(); + _hasher = new OptimizedCRC.OptimizedCRC(); break; case Hash.MD5: @@ -78,7 +77,7 @@ namespace SabreTools.Library.Tools switch (HashType) { case Hash.CRC: - (_hasher as OptimizedCRC).Update(buffer, 0, size); + (_hasher as OptimizedCRC.OptimizedCRC).Update(buffer, 0, size); break; case Hash.MD5: @@ -107,7 +106,7 @@ namespace SabreTools.Library.Tools switch (HashType) { case Hash.CRC: - (_hasher as OptimizedCRC).Update(emptyBuffer, 0, 0); + (_hasher as OptimizedCRC.OptimizedCRC).Update(emptyBuffer, 0, 0); break; case Hash.MD5: @@ -135,7 +134,7 @@ namespace SabreTools.Library.Tools switch (HashType) { case Hash.CRC: - return BitConverter.GetBytes((_hasher as OptimizedCRC).Value).Reverse().ToArray(); + return BitConverter.GetBytes((_hasher as OptimizedCRC.OptimizedCRC).Value).Reverse().ToArray(); case Hash.MD5: #if NET_FRAMEWORK diff --git a/SabreTools.Library/External/OptimizedCRC.cs b/SabreTools.IO/OptimizedCRC.cs similarity index 98% rename from SabreTools.Library/External/OptimizedCRC.cs rename to SabreTools.IO/OptimizedCRC.cs index 6d533d65..160a7757 100644 --- a/SabreTools.Library/External/OptimizedCRC.cs +++ b/SabreTools.IO/OptimizedCRC.cs @@ -24,9 +24,9 @@ using System; -namespace SabreTools.Library.External +namespace OptimizedCRC { - public class OptimizedCRC : IDisposable + internal class OptimizedCRC : IDisposable { private const uint kCrcPoly = 0xEDB88320; private const uint kInitial = 0xFFFFFFFF; diff --git a/SabreTools.IO/StreamExtensions.cs b/SabreTools.IO/StreamExtensions.cs new file mode 100644 index 00000000..a56c7c3d --- /dev/null +++ b/SabreTools.IO/StreamExtensions.cs @@ -0,0 +1,66 @@ +using System; +using System.IO; +using System.Linq; + +using SabreTools.Logging; + +namespace SabreTools.IO +{ + /// + /// Extensions to Stream functionality + /// + public static class StreamExtensions + { + /// + /// Add an aribtrary number of bytes to the inputted stream + /// + /// Stream to be appended to + /// Outputted stream + /// Bytes to be added to head of stream + /// Bytes to be added to tail of stream + public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) + { + // Write out prepended bytes + if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0) + output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length); + + // Now copy the existing file over + input.CopyTo(output); + + // Write out appended bytes + if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0) + output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length); + } + + /// + /// Seek to a specific point in the stream, if possible + /// + /// Input stream to try seeking on + /// Optional offset to seek to + public static long SeekIfPossible(this Stream input, long offset = 0) + { + try + { + if (input.CanSeek) + { + if (offset < 0) + return input.Seek(offset, SeekOrigin.End); + else if (offset >= 0) + return input.Seek(offset, SeekOrigin.Begin); + } + + return input.Position; + } + catch (NotSupportedException ex) + { + LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed"); + } + catch (NotImplementedException ex) + { + LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed"); + } + + return -1; + } + } +} diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index b31f646a..5223b218 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -2779,7 +2779,7 @@ namespace SabreTools.Library.DatFiles if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true)) { // Get the file informations that we will be using - Rom headerless = new Rom(transformStream.GetInfo(keepReadOpen: true)); + Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true)); // If we have duplicates and we're not filtering if (ShouldRebuild(headerless, transformStream, false, out dupes)) @@ -2851,7 +2851,7 @@ namespace SabreTools.Library.DatFiles string machinename = null; // Get the item from the current file - Rom item = new Rom(stream.GetInfo(keepReadOpen: true)); + Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true)); item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name); item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name); diff --git a/SabreTools.Library/DatFiles/Enums.cs b/SabreTools.Library/DatFiles/Enums.cs index 2ac887c0..d00dad72 100644 --- a/SabreTools.Library/DatFiles/Enums.cs +++ b/SabreTools.Library/DatFiles/Enums.cs @@ -181,34 +181,6 @@ namespace SabreTools.Library.DatFiles SHA512, } - /// - /// Available hashing types - /// - [Flags] - public enum Hash - { - CRC = 1 << 0, - MD5 = 1 << 1, -#if NET_FRAMEWORK - RIPEMD160 = 1 << 2, -#endif - SHA1 = 1 << 3, - SHA256 = 1 << 4, - SHA384 = 1 << 5, - SHA512 = 1 << 6, - SpamSum = 1 << 7, - - // Special combinations - Standard = CRC | MD5 | SHA1, -#if NET_FRAMEWORK - DeepHashes = RIPEMD160 | SHA256 | SHA384 | SHA512 | SpamSum, - SecureHashes = MD5 | RIPEMD160 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, -#else - DeepHashes = SHA256 | SHA384 | SHA512 | SpamSum, - SecureHashes = MD5 | SHA1 | SHA256 | SHA384 | SHA512 | SpamSum, -#endif - } - /// /// Determines merging tag handling for DAT output /// diff --git a/SabreTools.Library/DatFiles/Hashfile.cs b/SabreTools.Library/DatFiles/Hashfile.cs index 20af0744..d0c6667a 100644 --- a/SabreTools.Library/DatFiles/Hashfile.cs +++ b/SabreTools.Library/DatFiles/Hashfile.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using SabreTools.Data; using SabreTools.IO; using SabreTools.Library.DatItems; using SabreTools.Library.IO; diff --git a/SabreTools.Library/FileTypes/AaruFormat.cs b/SabreTools.Library/FileTypes/AaruFormat.cs index a90d27b5..d7ce03bd 100644 --- a/SabreTools.Library/FileTypes/AaruFormat.cs +++ b/SabreTools.Library/FileTypes/AaruFormat.cs @@ -2,6 +2,7 @@ using System.IO; using System.Text; +using SabreTools.IO; using SabreTools.Data; using SabreTools.Library.FileTypes.Aaru; using SabreTools.Library.IO; diff --git a/SabreTools.Library/FileTypes/BaseFile.cs b/SabreTools.Library/FileTypes/BaseFile.cs index 7b01f285..f79d0697 100644 --- a/SabreTools.Library/FileTypes/BaseFile.cs +++ b/SabreTools.Library/FileTypes/BaseFile.cs @@ -1,7 +1,13 @@ -using System.IO; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; -using SabreTools.Library.DatFiles; +using SabreTools.Data; +using SabreTools.IO; using SabreTools.Library.IO; +using SabreTools.Logging; +using Compress.ThreadReaders; namespace SabreTools.Library.FileTypes { @@ -135,7 +141,7 @@ namespace SabreTools.Library.FileTypes if (getHashes) { - BaseFile temp = stream.GetInfo(hashes: this.AvailableHashes); + BaseFile temp = GetInfo(stream, hashes: this.AvailableHashes); if (temp != null) { this.Parent = temp.Parent; @@ -156,5 +162,131 @@ namespace SabreTools.Library.FileTypes } #endregion + + #region Static Methods + + /// + /// Retrieve file information for a single file + /// + /// Filename to get information from + /// Size of the input stream + /// Hashes to include in the information + /// True if the underlying read stream should be kept open, false otherwise + /// Populated BaseFile object if success, empty one on error + public static BaseFile GetInfo(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false) + { + // If we want to automatically set the size + if (size == -1) + size = input.Length; + + try + { + // Get a list of hashers to run over the buffer + List hashers = new List(); + + if (hashes.HasFlag(Hash.CRC)) + hashers.Add(new Hasher(Hash.CRC)); + if (hashes.HasFlag(Hash.MD5)) + hashers.Add(new Hasher(Hash.MD5)); +#if NET_FRAMEWORK + if (hashes.HasFlag(Hash.RIPEMD160)) + hashers.Add(new Hasher(Hash.RIPEMD160)); +#endif + if (hashes.HasFlag(Hash.SHA1)) + hashers.Add(new Hasher(Hash.SHA1)); + if (hashes.HasFlag(Hash.SHA256)) + hashers.Add(new Hasher(Hash.SHA256)); + if (hashes.HasFlag(Hash.SHA384)) + hashers.Add(new Hasher(Hash.SHA384)); + if (hashes.HasFlag(Hash.SHA512)) + hashers.Add(new Hasher(Hash.SHA512)); + if (hashes.HasFlag(Hash.SpamSum)) + hashers.Add(new Hasher(Hash.SpamSum)); + + // Initialize the hashing helpers + var loadBuffer = new ThreadLoadBuffer(input); + int buffersize = 3 * 1024 * 1024; + byte[] buffer0 = new byte[buffersize]; + byte[] buffer1 = new byte[buffersize]; + + /* + Please note that some of the following code is adapted from + RomVault. This is a modified version of how RomVault does + threaded hashing. As such, some of the terminology and code + is the same, though variable names and comments may have + been tweaked to better fit this code base. + */ + + // Pre load the first buffer + long refsize = size; + int next = refsize > buffersize ? buffersize : (int)refsize; + input.Read(buffer0, 0, next); + int current = next; + refsize -= next; + bool bufferSelect = true; + + while (current > 0) + { + // Trigger the buffer load on the second buffer + next = refsize > buffersize ? buffersize : (int)refsize; + if (next > 0) + loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); + + byte[] buffer = bufferSelect ? buffer0 : buffer1; + + // Run hashes in parallel + Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current)); + + // Wait for the load buffer worker, if needed + if (next > 0) + loadBuffer.Wait(); + + // Setup for the next hashing step + current = next; + refsize -= next; + bufferSelect = !bufferSelect; + } + + // Finalize all hashing helpers + loadBuffer.Finish(); + Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize()); + + // Get the results + BaseFile baseFile = new BaseFile() + { + Size = size, + CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null, + MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null, +#if NET_FRAMEWORK + RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null, +#endif + SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null, + SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null, + SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null, + SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null, + SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null, + }; + + // Dispose of the hashers + loadBuffer.Dispose(); + hashers.ForEach(h => h.Dispose()); + + return baseFile; + } + catch (IOException ex) + { + LoggerImpl.Warning(ex, "An exception occurred during hashing."); + return new BaseFile(); + } + finally + { + if (!keepReadOpen) + input.Dispose(); + else + input.SeekIfPossible(); + } + } + + #endregion } } diff --git a/SabreTools.Library/FileTypes/GZipArchive.cs b/SabreTools.Library/FileTypes/GZipArchive.cs index 24840654..64d5e472 100644 --- a/SabreTools.Library/FileTypes/GZipArchive.cs +++ b/SabreTools.Library/FileTypes/GZipArchive.cs @@ -239,7 +239,7 @@ namespace SabreTools.Library.FileTypes var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); - gzipEntryRom = gzstream.GetInfo(hashes: this.AvailableHashes); + gzipEntryRom = GetInfo(gzstream, hashes: this.AvailableHashes); gzipEntryRom.Filename = gz.Filename(0); gzipEntryRom.Parent = gamename; gzipEntryRom.Date = (gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null); @@ -461,7 +461,7 @@ namespace SabreTools.Library.FileTypes outDir = Path.GetFullPath(outDir); // Now get the Rom info for the file so we have hashes and size - rom = new Rom(inputStream.GetInfo(keepReadOpen: true)); + rom = new Rom(GetInfo(inputStream, keepReadOpen: true)); // Get the output file name string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth)); diff --git a/SabreTools.Library/FileTypes/RarArchive.cs b/SabreTools.Library/FileTypes/RarArchive.cs index 253724cd..c5146268 100644 --- a/SabreTools.Library/FileTypes/RarArchive.cs +++ b/SabreTools.Library/FileTypes/RarArchive.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using SabreTools.Library.DatFiles; +using SabreTools.Data; using SabreTools.Library.DatItems; using SabreTools.Library.IO; using SharpCompress.Archives; @@ -199,7 +199,7 @@ namespace SabreTools.Library.FileTypes { using (Stream entryStream = entry.OpenEntryStream()) { - rarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes); + rarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes); } } diff --git a/SabreTools.Library/FileTypes/SevenZipArchive.cs b/SabreTools.Library/FileTypes/SevenZipArchive.cs index 1428e8f0..7ef30f97 100644 --- a/SabreTools.Library/FileTypes/SevenZipArchive.cs +++ b/SabreTools.Library/FileTypes/SevenZipArchive.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using SabreTools.Library.DatFiles; +using SabreTools.Data; using SabreTools.Library.DatItems; using SabreTools.Library.IO; using SabreTools.Library.Tools; @@ -307,7 +307,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true); + zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true); } // Fill in comon details and add to the list diff --git a/SabreTools.Library/FileTypes/TapeArchive.cs b/SabreTools.Library/FileTypes/TapeArchive.cs index 6154e0b3..d52a45b5 100644 --- a/SabreTools.Library/FileTypes/TapeArchive.cs +++ b/SabreTools.Library/FileTypes/TapeArchive.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using SabreTools.Library.DatFiles; +using SabreTools.Data; using SabreTools.Library.DatItems; using SabreTools.Library.IO; using SabreTools.Library.Tools; @@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes { using (Stream entryStream = entry.OpenEntryStream()) { - tarEntryRom = entryStream.GetInfo(size: entry.Size, hashes: this.AvailableHashes); + tarEntryRom = GetInfo(entryStream, size: entry.Size, hashes: this.AvailableHashes); } } diff --git a/SabreTools.Library/FileTypes/XZArchive.cs b/SabreTools.Library/FileTypes/XZArchive.cs index 9fef7699..9bd17507 100644 --- a/SabreTools.Library/FileTypes/XZArchive.cs +++ b/SabreTools.Library/FileTypes/XZArchive.cs @@ -229,7 +229,7 @@ namespace SabreTools.Library.FileTypes else { var xzStream = new XZStream(File.OpenRead(this.Filename)); - xzEntryRom = xzStream.GetInfo(hashes: this.AvailableHashes); + xzEntryRom = GetInfo(xzStream, hashes: this.AvailableHashes); xzEntryRom.Filename = gamename; xzStream.Dispose(); } @@ -360,7 +360,7 @@ namespace SabreTools.Library.FileTypes outDir = Path.GetFullPath(outDir); // Now get the Rom info for the file so we have hashes and size - rom = new Rom(inputStream.GetInfo(keepReadOpen: true)); + rom = new Rom(GetInfo(inputStream, keepReadOpen: true)); // Get the output file name string outfile = Path.Combine(outDir, PathExtensions.GetDepotPath(rom.SHA1, Depth)); diff --git a/SabreTools.Library/FileTypes/ZipArchive.cs b/SabreTools.Library/FileTypes/ZipArchive.cs index cd7a1632..cef92461 100644 --- a/SabreTools.Library/FileTypes/ZipArchive.cs +++ b/SabreTools.Library/FileTypes/ZipArchive.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using SabreTools.Library.DatFiles; +using SabreTools.Data; using SabreTools.Library.DatItems; using SabreTools.Library.IO; using SabreTools.Library.Tools; @@ -308,7 +308,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true); + zipEntryRom = GetInfo(readStream, size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true); } // Fill in comon details and add to the list diff --git a/SabreTools.Library/IO/FileExtensions.cs b/SabreTools.Library/IO/FileExtensions.cs index cb23aac8..c2775d26 100644 --- a/SabreTools.Library/IO/FileExtensions.cs +++ b/SabreTools.Library/IO/FileExtensions.cs @@ -391,7 +391,7 @@ namespace SabreTools.Library.IO else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) baseFile = CHDFile.Create(inputStream); else - baseFile = inputStream.GetInfo(hashes: hashes, keepReadOpen: false); + baseFile = BaseFile.GetInfo(inputStream, hashes: hashes, keepReadOpen: false); // Dispose of the input stream inputStream?.Dispose(); diff --git a/SabreTools.Library/IO/StreamExtensions.cs b/SabreTools.Library/IO/StreamExtensions.cs deleted file mode 100644 index a51658ea..00000000 --- a/SabreTools.Library/IO/StreamExtensions.cs +++ /dev/null @@ -1,208 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - -using SabreTools.Data; -using SabreTools.Logging; -using SabreTools.Library.DatFiles; -using SabreTools.Library.FileTypes; -using SabreTools.Library.Tools; -using Compress.ThreadReaders; - -namespace SabreTools.Library.IO -{ - /// - /// Extensions to Stream functionality - /// - public static class StreamExtensions - { - /// - /// Add an aribtrary number of bytes to the inputted stream - /// - /// Stream to be appended to - /// Outputted stream - /// Bytes to be added to head of stream - /// Bytes to be added to tail of stream - public static void AppendBytes(Stream input, Stream output, byte[] bytesToAddToHead, byte[] bytesToAddToTail) - { - // Write out prepended bytes - if (bytesToAddToHead != null && bytesToAddToHead.Count() > 0) - output.Write(bytesToAddToHead, 0, bytesToAddToHead.Length); - - // Now copy the existing file over - input.CopyTo(output); - - // Write out appended bytes - if (bytesToAddToTail != null && bytesToAddToTail.Count() > 0) - output.Write(bytesToAddToTail, 0, bytesToAddToTail.Length); - } - - /// - /// Retrieve file information for a single file - /// - /// Filename to get information from - /// Size of the input stream - /// Hashes to include in the information - /// True if the underlying read stream should be kept open, false otherwise - /// Populated BaseFile object if success, empty one on error - public static BaseFile GetInfo(this Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false) - { - return GetInfoAsync(input, size, hashes, keepReadOpen).ConfigureAwait(false).GetAwaiter().GetResult(); - } - - /// - /// Retrieve file information for a single file - /// - /// Filename to get information from - /// Size of the input stream - /// Hashes to include in the information - /// True if the underlying read stream should be kept open, false otherwise - /// Populated BaseFile object if success, empty one on error - public static async Task GetInfoAsync(Stream input, long size = -1, Hash hashes = Hash.Standard, bool keepReadOpen = false) - { - // If we want to automatically set the size - if (size == -1) - size = input.Length; - - try - { - // Get a list of hashers to run over the buffer - List hashers = new List(); - - if (hashes.HasFlag(Hash.CRC)) - hashers.Add(new Hasher(Hash.CRC)); - if (hashes.HasFlag(Hash.MD5)) - hashers.Add(new Hasher(Hash.MD5)); -#if NET_FRAMEWORK - if (hashes.HasFlag(Hash.RIPEMD160)) - hashers.Add(new Hasher(Hash.RIPEMD160)); -#endif - if (hashes.HasFlag(Hash.SHA1)) - hashers.Add(new Hasher(Hash.SHA1)); - if (hashes.HasFlag(Hash.SHA256)) - hashers.Add(new Hasher(Hash.SHA256)); - if (hashes.HasFlag(Hash.SHA384)) - hashers.Add(new Hasher(Hash.SHA384)); - if (hashes.HasFlag(Hash.SHA512)) - hashers.Add(new Hasher(Hash.SHA512)); - if (hashes.HasFlag(Hash.SpamSum)) - hashers.Add(new Hasher(Hash.SpamSum)); - - // Initialize the hashing helpers - var loadBuffer = new ThreadLoadBuffer(input); - int buffersize = 3 * 1024 * 1024; - byte[] buffer0 = new byte[buffersize]; - byte[] buffer1 = new byte[buffersize]; - - /* - Please note that some of the following code is adapted from - RomVault. This is a modified version of how RomVault does - threaded hashing. As such, some of the terminology and code - is the same, though variable names and comments may have - been tweaked to better fit this code base. - */ - - // Pre load the first buffer - long refsize = size; - int next = refsize > buffersize ? buffersize : (int)refsize; - input.Read(buffer0, 0, next); - int current = next; - refsize -= next; - bool bufferSelect = true; - - while (current > 0) - { - // Trigger the buffer load on the second buffer - next = refsize > buffersize ? buffersize : (int)refsize; - if (next > 0) - loadBuffer.Trigger(bufferSelect ? buffer1 : buffer0, next); - - byte[] buffer = bufferSelect ? buffer0 : buffer1; - - // Run hashes in parallel - Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Process(buffer, current)); - - // Wait for the load buffer worker, if needed - if (next > 0) - loadBuffer.Wait(); - - // Setup for the next hashing step - current = next; - refsize -= next; - bufferSelect = !bufferSelect; - } - - // Finalize all hashing helpers - loadBuffer.Finish(); - Parallel.ForEach(hashers, Globals.ParallelOptions, h => h.Finalize()); - - // Get the results - BaseFile baseFile = new BaseFile() - { - Size = size, - CRC = hashes.HasFlag(Hash.CRC) ? hashers.First(h => h.HashType == Hash.CRC).GetHash() : null, - MD5 = hashes.HasFlag(Hash.MD5) ? hashers.First(h => h.HashType == Hash.MD5).GetHash() : null, -#if NET_FRAMEWORK - RIPEMD160 = hashes.HasFlag(Hash.RIPEMD160) ? hashers.First(h => h.HashType == Hash.RIPEMD160).GetHash() : null, -#endif - SHA1 = hashes.HasFlag(Hash.SHA1) ? hashers.First(h => h.HashType == Hash.SHA1).GetHash() : null, - SHA256 = hashes.HasFlag(Hash.SHA256) ? hashers.First(h => h.HashType == Hash.SHA256).GetHash() : null, - SHA384 = hashes.HasFlag(Hash.SHA384) ? hashers.First(h => h.HashType == Hash.SHA384).GetHash() : null, - SHA512 = hashes.HasFlag(Hash.SHA512) ? hashers.First(h => h.HashType == Hash.SHA512).GetHash() : null, - SpamSum = hashes.HasFlag(Hash.SpamSum) ? hashers.First(h => h.HashType == Hash.SpamSum).GetHash() : null, - }; - - // Dispose of the hashers - loadBuffer.Dispose(); - hashers.ForEach(h => h.Dispose()); - - return baseFile; - } - catch (IOException ex) - { - LoggerImpl.Warning(ex, "An exception occurred during hashing."); - return new BaseFile(); - } - finally - { - if (!keepReadOpen) - input.Dispose(); - else - input.SeekIfPossible(); - } - } - - /// - /// Seek to a specific point in the stream, if possible - /// - /// Input stream to try seeking on - /// Optional offset to seek to - public static long SeekIfPossible(this Stream input, long offset = 0) - { - try - { - if (input.CanSeek) - { - if (offset < 0) - return input.Seek(offset, SeekOrigin.End); - else if (offset >= 0) - return input.Seek(offset, SeekOrigin.Begin); - } - - return input.Position; - } - catch (NotSupportedException ex) - { - LoggerImpl.Verbose(ex, "Stream does not support seeking to starting offset. Stream position not changed"); - } - catch (NotImplementedException ex) - { - LoggerImpl.Warning(ex, "Stream does not support seeking to starting offset. Stream position not changed"); - } - - return -1; - } - } -} diff --git a/SabreTools.Library/Tools/Converters.cs b/SabreTools.Library/Tools/Converters.cs index e0954d41..4d290b36 100644 --- a/SabreTools.Library/Tools/Converters.cs +++ b/SabreTools.Library/Tools/Converters.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Text.RegularExpressions; +using SabreTools.Data; using SabreTools.Library.DatFiles; using SabreTools.Library.DatItems; using SabreTools.Library.FileTypes; diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index 9af19c93..e80e826c 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using SabreTools.Data; using SabreTools.Help; using SabreTools.IO; using SabreTools.Library.DatFiles; diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 6f951368..7dd0b044 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; +using SabreTools.Data; using SabreTools.Help; using SabreTools.IO; using SabreTools.Library.DatFiles;