using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatItems; using SabreTools.FileTypes; using SabreTools.IO; using SabreTools.Logging; using SabreTools.Skippers; // This file represents all methods related to rebuilding from a DatFile namespace SabreTools.DatFiles { public abstract partial class DatFile { /// /// Process the DAT and find all matches in input files and folders assuming they're a depot /// /// List of input files/folders to check /// Output directory to use to build to /// True if the date from the DAT should be used if available, false otherwise /// True if input files should be deleted, false otherwise /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output format that files should be written to /// True if rebuilding was a success, false otherwise public bool RebuildDepot( List inputs, string outDir, bool date = false, bool delete = false, bool inverse = false, OutputFormat outputFormat = OutputFormat.Folder) { #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit if (Items.TotalCount == 0 && !inverse) { logger.User("No entries were found to rebuild, exiting..."); return false; } // Check that the output directory exists outDir = DirectoryExtensions.Ensure(outDir, create: true); // Now we want to get forcepack flag if it's not overridden if (outputFormat == OutputFormat.Folder && Header.ForcePacking != PackingFlag.None) outputFormat = GetOutputFormat(Header.ForcePacking); // Preload the Skipper list SkipperMatch.Init(); #endregion bool success = true; #region Rebuild from depots in order string format = FromOutputFormat(outputFormat) ?? string.Empty; InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}"); // Now loop through and get only directories from the input paths List directories = new List(); Parallel.ForEach(inputs, Globals.ParallelOptions, input => { // Add to the list if the input is a directory if (Directory.Exists(input)) { logger.Verbose($"Adding depot: {input}"); lock (directories) { directories.Add(input); } } }); // If we don't have any directories, we want to exit if (directories.Count == 0) return success; // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 Items.BucketBy(Field.DatItem_SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild var keys = Items.SortedKeys.ToList(); foreach (string hash in keys) { // Pre-empt any issues that could arise from string length if (hash.Length != Constants.SHA1Length) continue; logger.User($"Checking hash '{hash}'"); // Get the extension path for the hash string subpath = PathExtensions.GetDepotPath(hash, Header.InputDepot.Depth); // Find the first depot that includes the hash string foundpath = null; foreach (string directory in directories) { if (File.Exists(Path.Combine(directory, subpath))) { foundpath = Path.Combine(directory, subpath); break; } } // If we didn't find a path, then we continue if (foundpath == null) continue; // If we have a path, we want to try to get the rom information GZipArchive archive = new GZipArchive(foundpath); BaseFile fileinfo = archive.GetTorrentGZFileInfo(); // If the file information is null, then we continue if (fileinfo == null) continue; // Ensure we are sorted correctly (some other calls can change this) Items.BucketBy(Field.DatItem_SHA1, DedupeType.None); // If there are no items in the hash, we continue if (Items[hash] == null || Items[hash].Count == 0) continue; // Otherwise, we rebuild that file to all locations that we need to bool usedInternally; if (Items[hash][0].ItemType == ItemType.Disk) usedInternally = RebuildIndividualFile(new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); else if (Items[hash][0].ItemType == ItemType.Media) usedInternally = RebuildIndividualFile(new Media(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); else usedInternally = RebuildIndividualFile(new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); // If we are supposed to delete the depot file, do so if (delete && usedInternally) File.Delete(foundpath); } watch.Stop(); #endregion return success; } /// /// Process the DAT and find all matches in input files and folders /// /// List of input files/folders to check /// Output directory to use to build to /// True to enable external scanning of archives, false otherwise /// True if the date from the DAT should be used if available, false otherwise /// True if input files should be deleted, false otherwise /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output format that files should be written to /// TreatAsFiles representing special format scanning /// True if rebuilding was a success, false otherwise public bool RebuildGeneric( List inputs, string outDir, bool quickScan = false, bool date = false, bool delete = false, bool inverse = false, OutputFormat outputFormat = OutputFormat.Folder, TreatAsFile asFiles = 0x00) { #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit if (Items.TotalCount == 0 && !inverse) { logger.User("No entries were found to rebuild, exiting..."); return false; } // Check that the output directory exists if (!Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); outDir = Path.GetFullPath(outDir); } // Now we want to get forcepack flag if it's not overridden if (outputFormat == OutputFormat.Folder && Header.ForcePacking != PackingFlag.None) outputFormat = GetOutputFormat(Header.ForcePacking); // Preload the Skipper list SkipperMatch.Init(); #endregion bool success = true; #region Rebuild from sources in order string format = FromOutputFormat(outputFormat) ?? string.Empty; InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}"); // Now loop through all of the files in all of the inputs foreach (string input in inputs) { // If the input is a file if (File.Exists(input)) { logger.User($"Checking file: {input}"); bool rebuilt = RebuildGenericHelper(input, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) File.Delete(input); } // If the input is a directory else if (Directory.Exists(input)) { logger.Verbose($"Checking directory: {input}"); foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) { logger.User($"Checking file: {file}"); bool rebuilt = RebuildGenericHelper(file, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) File.Delete(input); } } } watch.Stop(); #endregion return success; } /// /// Attempt to add a file to the output if it matches /// /// Name of the file to process /// Output directory to use to build to /// True to enable external scanning of archives, false otherwise /// True if the date from the DAT should be used if available, false otherwise /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output format that files should be written to /// TreatAsFiles representing special format scanning /// True if the file was used to rebuild, false otherwise private bool RebuildGenericHelper( string file, string outDir, bool quickScan, bool date, bool inverse, OutputFormat outputFormat, TreatAsFile asFiles) { // If we somehow have a null filename, return if (file == null) return false; // Set the deletion variables bool usedExternally = false, usedInternally = false; // Create an empty list of BaseFile for archive entries List entries = null; // Get the TGZ and TXZ status for later GZipArchive tgz = new GZipArchive(file); XZArchive txz = new XZArchive(file); bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent(); // Get the base archive first BaseArchive archive = BaseArchive.Create(file); // Now get all extracted items from the archive if (archive != null) { archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard; entries = archive.GetChildren(); } // If the entries list is null, we encountered an error or have a file and should scan externally if (entries == null && File.Exists(file)) { BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles); // Create the correct DatItem DatItem internalDatItem; if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) internalDatItem = new Media(internalFileInfo); else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) internalDatItem = new Disk(internalFileInfo); else internalDatItem = new Rom(internalFileInfo); usedExternally = RebuildIndividualFile(internalDatItem, file, outDir, date, inverse, outputFormat); } // Otherwise, loop through the entries and try to match else { foreach (BaseFile entry in entries) { DatItem internalDatItem = DatItem.Create(entry); usedInternally |= RebuildIndividualFile(internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */); } } return usedExternally || usedInternally; } /// /// Find duplicates and rebuild individual files to output /// /// Information for the current file to rebuild from /// Name of the file to process /// Output directory to use to build to /// True if the date from the DAT should be used if available, false otherwise /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output format that files should be written to /// True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise /// True if the file was able to be rebuilt, false otherwise private bool RebuildIndividualFile( DatItem datItem, string file, string outDir, bool date, bool inverse, OutputFormat outputFormat, bool? isZip = null) { // Set the initial output value bool rebuilt = false; // If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) && !(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) && !(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba)) { outputFormat = OutputFormat.Folder; } // If we have a Disk or Media, change it into a Rom for later use if (datItem.ItemType == ItemType.Disk) datItem = (datItem as Disk).ConvertToRom(); else if (datItem.ItemType == ItemType.Media) datItem = (datItem as Media).ConvertToRom(); // Prepopluate a key string string crc = (datItem as Rom).CRC ?? string.Empty; // Try to get the stream for the file if (!GetFileStream(datItem, file, isZip, out Stream fileStream)) return false; // If either we have duplicates or we're filtering if (ShouldRebuild(datItem, fileStream, inverse, out List dupes)) { // If we have a very specific TGZ->TGZ case, just copy it accordingly if (RebuildTorrentGzip(datItem, file, outDir, outputFormat, isZip)) return true; // If we have a very specific TXZ->TXZ case, just copy it accordingly if (RebuildTorrentXz(datItem, file, outDir, outputFormat, isZip)) return true; logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Special case for partial packing mode bool shouldCheck = false; if (outputFormat == OutputFormat.Folder && Header.ForcePacking == PackingFlag.Partial) { shouldCheck = true; Items.BucketBy(Field.Machine_Name, DedupeType.None, lower: false); } // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // If we should check for the items in the machine if (shouldCheck && Items[item.Machine.Name].Count > 1) outputFormat = OutputFormat.Folder; else if (shouldCheck && Items[item.Machine.Name].Count == 1) outputFormat = OutputFormat.ParentFolder; // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(date, outputFormat); // Now rebuild to the output file outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile()); } // Close the input stream fileStream?.Dispose(); } // Now we want to take care of headers, if applicable if (Header.HeaderSkipper != null) { // Check to see if we have a matching header first SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(Header.HeaderSkipper)); // If there's a match, create the new file to write if (rule.Tests != null && rule.Tests.Count != 0) { // If the file could be transformed correctly MemoryStream transformStream = new MemoryStream(); if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true)) { // Get the file informations that we will be using Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true)); // If we have duplicates and we're not filtering if (ShouldRebuild(headerless, transformStream, false, out dupes)) { logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // Create a headered item to use as well datItem.CopyMachineInformation(item); datItem.SetFields(new Dictionary { [Field.DatItem_Name] = $"{datItem.GetName()}_{crc}" }); // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(date, outputFormat); // Now rebuild to the output file bool eitherSuccess = false; eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile()); eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile()); // Now add the success of either rebuild rebuilt &= eitherSuccess; } } } // Dispose of the stream transformStream?.Dispose(); } // Dispose of the stream fileStream?.Dispose(); } return rebuilt; } /// /// Get the rebuild state for a given item /// /// Information for the current file to rebuild from /// Stream representing the input file /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output list of duplicate items to rebuild to /// True if the item should be rebuilt, false otherwise private bool ShouldRebuild(DatItem datItem, Stream stream, bool inverse, out List dupes) { // Find if the file has duplicates in the DAT dupes = Items.GetDuplicates(datItem); bool hasDuplicates = dupes.Count > 0; // If we have duplicates but we're filtering if (hasDuplicates && inverse) { return false; } // If we have duplicates without filtering else if (hasDuplicates && !inverse) { return true; } // If we have no duplicates and we're filtering else if (!hasDuplicates && inverse) { string machinename = null; // Get the item from the current file Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true)); item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name); item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name); // If we are coming from an archive, set the correct machine name if (machinename != null) { item.Machine.Name = machinename; item.Machine.Description = machinename; } dupes.Add(item); return true; } // If we have no duplicates and we're not filtering else { return false; } } /// /// Rebuild from TorrentGzip to TorrentGzip /// /// Information for the current file to rebuild from /// Name of the file to process /// Output directory to use to build to /// Output format that files should be written to /// True if the input file is an archive, false if the file is TGZ, null otherwise /// True if rebuilt properly, false otherwise private bool RebuildTorrentGzip(DatItem datItem, string file, string outDir, OutputFormat outputFormat, bool? isZip) { // If we have a very specific TGZ->TGZ case, just copy it accordingly GZipArchive tgz = new GZipArchive(file); BaseFile tgzRom = tgz.GetTorrentGZFileInfo(); if (isZip == false && tgzRom != null && (outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba)) { logger.User($"Matches found for '{Path.GetFileName(datItem.GetName() ?? string.Empty)}', rebuilding accordingly..."); // Get the proper output path string sha1 = (datItem as Rom).SHA1 ?? string.Empty; if (outputFormat == OutputFormat.TorrentGzipRomba) outDir = Path.Combine(outDir, PathExtensions.GetDepotPath(sha1, Header.OutputDepot.Depth)); else outDir = Path.Combine(outDir, sha1 + ".gz"); // Make sure the output folder is created Directory.CreateDirectory(Path.GetDirectoryName(outDir)); // Now copy the file over try { File.Copy(file, outDir); return true; } catch { return false; } } return false; } /// /// Rebuild from TorrentXz to TorrentXz /// /// Information for the current file to rebuild from /// Name of the file to process /// Output directory to use to build to /// Output format that files should be written to /// True if the input file is an archive, false if the file is TXZ, null otherwise /// True if rebuilt properly, false otherwise private bool RebuildTorrentXz(DatItem datItem, string file, string outDir, OutputFormat outputFormat, bool? isZip) { // If we have a very specific TGZ->TGZ case, just copy it accordingly XZArchive txz = new XZArchive(file); BaseFile txzRom = txz.GetTorrentXZFileInfo(); if (isZip == false && txzRom != null && (outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba)) { logger.User($"Matches found for '{Path.GetFileName(datItem.GetName() ?? string.Empty)}', rebuilding accordingly..."); // Get the proper output path string sha1 = (datItem as Rom).SHA1 ?? string.Empty; if (outputFormat == OutputFormat.TorrentXZRomba) outDir = Path.Combine(outDir, PathExtensions.GetDepotPath(sha1, Header.OutputDepot.Depth)).Replace(".gz", ".xz"); else outDir = Path.Combine(outDir, sha1 + ".xz"); // Make sure the output folder is created Directory.CreateDirectory(Path.GetDirectoryName(outDir)); // Now copy the file over try { File.Copy(file, outDir); return true; } catch { return false; } } return false; } /// /// Get the Stream related to a file /// /// Information for the current file to rebuild from /// Name of the file to process /// Non-null if the input file is an archive /// Output stream representing the opened file /// True if the stream opening succeeded, false otherwise private bool GetFileStream(DatItem datItem, string file, bool? isZip, out Stream stream) { // Get a generic stream for the file stream = null; // If we have a zipfile, extract the stream to memory if (isZip != null) { BaseArchive archive = BaseArchive.Create(file); if (archive != null) (stream, _) = archive.CopyToStream(datItem.GetName() ?? datItem.ItemType.ToString()); } // Otherwise, just open the filestream else { stream = File.OpenRead(file); } // If the stream is null, then continue if (stream == null) return false; // Seek to the beginning of the stream if (stream.CanSeek) stream.Seek(0, SeekOrigin.Begin); return true; } /// /// Get the default OutputFormat associated with each PackingFlag /// private OutputFormat GetOutputFormat(PackingFlag packing) { #if NET_FRAMEWORK switch (packing) { case PackingFlag.Zip: return OutputFormat.TorrentZip; case PackingFlag.Unzip: case PackingFlag.Partial: return OutputFormat.Folder; case PackingFlag.Flat: return OutputFormat.ParentFolder; case PackingFlag.None: default: return OutputFormat.Folder; } #else return packing switch { PackingFlag.Zip => OutputFormat.TorrentZip, PackingFlag.Unzip => OutputFormat.Folder, PackingFlag.Partial => OutputFormat.Folder, PackingFlag.Flat => OutputFormat.ParentFolder, PackingFlag.None => OutputFormat.Folder, _ => OutputFormat.Folder, }; #endif } /// /// Get preconfigured Folder for rebuilding /// /// True if the date from the DAT should be used if available, false otherwise /// Output format that files should be written to /// Folder configured with proper flags private Folder GetPreconfiguredFolder(bool date, OutputFormat outputFormat) { Folder outputArchive = Folder.Create(outputFormat); if (outputArchive is BaseArchive baseArchive && date) baseArchive.UseDates = date; // Set the depth fields where appropriate if (outputArchive is GZipArchive gzipArchive) gzipArchive.Depth = Header.OutputDepot.Depth; else if (outputArchive is XZArchive xzArchive) xzArchive.Depth = Header.OutputDepot.Depth; return outputArchive; } /// /// Get string value from input OutputFormat /// /// OutputFormat to get value from /// String value corresponding to the OutputFormat private string FromOutputFormat(OutputFormat itemType) { #if NET_FRAMEWORK switch (itemType) { case OutputFormat.Folder: case OutputFormat.ParentFolder: return "directory"; case OutputFormat.TapeArchive: return "TAR"; case OutputFormat.Torrent7Zip: return "Torrent7Z"; case OutputFormat.TorrentGzip: case OutputFormat.TorrentGzipRomba: return "TorrentGZ"; case OutputFormat.TorrentLRZip: return "TorrentLRZ"; case OutputFormat.TorrentRar: return "TorrentRAR"; case OutputFormat.TorrentXZ: case OutputFormat.TorrentXZRomba: return "TorrentXZ"; case OutputFormat.TorrentZip: return "TorrentZip"; default: return null; } #else return itemType switch { OutputFormat.Folder => "directory", OutputFormat.ParentFolder => "directory", OutputFormat.TapeArchive => "TAR", OutputFormat.Torrent7Zip => "Torrent7Z", OutputFormat.TorrentGzip => "TorrentGZ", OutputFormat.TorrentGzipRomba => "TorrentGZ", OutputFormat.TorrentLRZip => "TorrentLRZ", OutputFormat.TorrentRar => "TorrentRAR", OutputFormat.TorrentXZ => "TorrentXZ", OutputFormat.TorrentXZRomba => "TorrentXZ", OutputFormat.TorrentZip => "TorrentZip", _ => null, }; #endif } } }