claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split DatFiles namespace

Browse Source
This commit is contained in:
Matt Nadareski
2020-12-10 23:24:09 -08:00
parent 24e73489d2
commit 24d4be0571
37 changed files with 426 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
SabreTools.DatFiles/DatFile.cs
View File

@@ -16,17 +16,8 @@ namespace SabreTools.DatFiles
/// <summary>
/// Represents a format-agnostic DAT
/// </summary>
/// <remarks>
/// The fact that this one class could be separated into as many partial
/// classes as it did means that the functionality here should probably
/// be split out into either separate classes or even an entirely separate
/// namespace. Also, with that in mind, each of the individual DatFile types
/// probably should only need to inherit from a thin abstract class and
/// should not be exposed as part of the library, instead being taken care
/// of behind the scenes as part of the reading and writing.
/// </remarks>
[JsonObject("datfile"), XmlRoot("datfile")]
public abstract partial class DatFile
public abstract class DatFile
{
#region Fields

450
SabreTools.DatFiles/DatFromDir.cs
View File

@@ -1,450 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.IO;
using SabreTools.Logging;
namespace SabreTools.DatFiles
{
/// <summary>
/// This file represents all methods related to populating a DatFile
/// from a set of files and directories
/// </summary>
public class DatFromDir
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Create a new Dat from a directory
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
public static bool PopulateFromDir(
DatFile datFile,
string basePath,
TreatAsFile asFiles = 0x00,
SkipFileType skipFileType = SkipFileType.None,
bool addBlanks = false,
Hash hashes = Hash.Standard)
{
// Clean the temp directory path
Globals.TempDir = Globals.TempDir.Ensure(temp: true);
// Set the progress variables
long totalSize = 0;
long currentSize = 0;
// Process the input
if (Directory.Exists(basePath))
{
logger.Verbose($"Folder found: {basePath}");
// Get a list of all files to process
List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList();
// Loop through and add the file sizes
Parallel.ForEach(files, Globals.ParallelOptions, item =>
{
Interlocked.Add(ref totalSize, new FileInfo(item).Length);
});
// Process the files in the main folder or any subfolder
logger.User(totalSize, currentSize);
foreach (string item in files)
{
CheckFileForHashes(datFile, item, basePath, asFiles, skipFileType, addBlanks, hashes);
currentSize += new FileInfo(item).Length;
logger.User(totalSize, currentSize, item);
}
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
ProcessDirectoryBlanks(datFile, basePath);
}
else if (File.Exists(basePath))
{
logger.Verbose($"File found: {basePath}");
totalSize = new FileInfo(basePath).Length;
logger.User(totalSize, currentSize);
string parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(datFile, basePath, parentPath, asFiles, skipFileType, addBlanks, hashes);
logger.User(totalSize, totalSize, basePath);
}
// Now that we're done, delete the temp folder (if it's not the default)
logger.User("Cleaning temp folder");
if (Globals.TempDir != Path.GetTempPath())
{
if (Directory.Exists(Globals.TempDir))
Directory.Delete(Globals.TempDir, true);
}
return true;
}
/// <summary>
/// Check a given file for hashes, based on current settings
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">Filename of the item to be checked</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
private static void CheckFileForHashes(
DatFile datFile,
string item,
string basePath,
TreatAsFile asFiles,
SkipFileType skipFileType,
bool addBlanks,
Hash hashes)
{
// If we're in depot mode, process it separately
if (CheckDepotFile(datFile, item))
return;
// Initialize possible archive variables
BaseArchive archive = BaseArchive.Create(item);
// Process archives according to flags
if (archive != null)
{
// Set the archive flags
archive.AvailableHashes = hashes;
// Skip if we're treating archives as files and skipping files
if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File)
{
return;
}
// Skip if we're skipping archives
else if (skipFileType == SkipFileType.Archive)
{
return;
}
// Process as archive if we're not treating archives as files
else if (!asFiles.HasFlag(TreatAsFile.Archive))
{
var extracted = archive.GetChildren();
// If we have internal items to process, do so
if (extracted != null)
ProcessArchive(datFile, item, basePath, extracted);
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
ProcessArchiveBlanks(datFile, item, basePath, archive);
}
// Process as file if we're treating archives as files
else
{
ProcessFile(datFile, item, basePath, hashes, asFiles);
}
}
// Process non-archives according to flags
else
{
// Skip if we're skipping files
if (skipFileType == SkipFileType.File)
return;
// Process as file
else
ProcessFile(datFile, item, basePath, hashes, asFiles);
}
}
/// <summary>
/// Check an item as if it's supposed to be in a depot
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">Filename of the item to be checked</param>
/// <returns>True if we checked a depot file, false otherwise</returns>
private static bool CheckDepotFile(DatFile datFile, string item)
{
// If we're not in Depot mode, return false
if (datFile.Header.OutputDepot?.IsActive != true)
return false;
// Check the file as if it were in a depot
GZipArchive gzarc = new GZipArchive(item);
BaseFile baseFile = gzarc.GetTorrentGZFileInfo();
// If the rom is valid, add it
if (baseFile != null && baseFile.Filename != null)
{
// Add the list if it doesn't exist already
Rom rom = new Rom(baseFile);
datFile.Items.Add(rom.GetKey(Field.DatItem_CRC), rom);
logger.Verbose($"File added: {Path.GetFileNameWithoutExtension(item)}");
}
else
{
logger.Verbose($"File not added: {Path.GetFileNameWithoutExtension(item)}");
return true;
}
return true;
}
/// <summary>
/// Process a single file as an archive
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="extracted">List of BaseFiles representing the internal files</param>
private static void ProcessArchive(DatFile datFile, string item, string basePath, List<BaseFile> extracted)
{
// Get the parent path for all items
string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item);
// First take care of the found items
Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile =>
{
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, parent);
});
}
/// <summary>
/// Process blank folders in an archive
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File containing the blanks</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="archive">BaseArchive to get blanks from</param>
private static void ProcessArchiveBlanks(DatFile datFile, string item, string basePath, BaseArchive archive)
{
List<string> empties = new List<string>();
// Get the parent path for all items
string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item);
// Now get all blank folders from the archive
if (archive != null)
empties = archive.GetEmptyFolders();
// Add add all of the found empties to the DAT
Parallel.ForEach(empties, Globals.ParallelOptions, empty =>
{
Rom emptyRom = new Rom(Path.Combine(empty, "_"), item);
ProcessFileHelper(datFile, item, emptyRom, basePath, parent);
});
}
/// <summary>
/// Process blank folders in a directory
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="basePath">Path the represents the parent directory</param>
private static void ProcessDirectoryBlanks(DatFile datFile, string basePath)
{
// If we're in depot mode, we don't process blanks
if (datFile.Header.OutputDepot?.IsActive == true)
return;
List<string> empties = basePath.ListEmpty();
Parallel.ForEach(empties, Globals.ParallelOptions, dir =>
{
// Get the full path for the directory
string fulldir = Path.GetFullPath(dir);
// Set the temporary variables
string gamename = string.Empty;
string romname = string.Empty;
// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
if (datFile.Header.Type == "SuperDAT")
{
gamename = fulldir.Remove(0, basePath.Length + 1);
romname = "_";
}
// Otherwise, we want just the top level folder as the game, and the file as everything else
else
{
gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0];
romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "_");
}
// Sanitize the names
gamename = gamename.Trim(Path.DirectorySeparatorChar);
romname = romname.Trim(Path.DirectorySeparatorChar);
logger.Verbose($"Adding blank empty folder: {gamename}");
datFile.Items["null"].Add(new Rom(romname, gamename));
});
}
/// <summary>
/// Process a single file as a file
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles)
{
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles);
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);
}
/// <summary>
/// Process a single file as a file (with found Rom data)
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="item">Rom data to be used to write to file</param>
/// <param name="basepath">Path the represents the parent directory</param>
/// <param name="parent">Parent game to be used</param>
private static void ProcessFileHelper(DatFile datFile, string item, DatItem datItem, string basepath, string parent)
{
// If we didn't get an accepted parsed type somehow, cancel out
List<ItemType> parsed = new List<ItemType> { ItemType.Disk, ItemType.Media, ItemType.Rom };
if (!parsed.Contains(datItem.ItemType))
return;
try
{
// If the basepath doesn't end with a directory separator, add it
if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
basepath += Path.DirectorySeparatorChar.ToString();
// Make sure we have the full item path
item = Path.GetFullPath(item);
// Process the item to sanitize names based on input
SetDatItemInfo(datFile, datItem, item, parent, basepath);
// Add the file information to the DAT
string key = datItem.GetKey(Field.DatItem_CRC);
datFile.Items.Add(key, datItem);
logger.Verbose($"File added: {datItem.GetName() ?? string.Empty}");
}
catch (IOException ex)
{
logger.Error(ex);
return;
}
}
/// <summary>
/// Set proper Game and Rom names from user inputs
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="datItem">DatItem representing the input file</param>
/// <param name="item">Item name to use</param>
/// <param name="parent">Parent name to use</param>
/// <param name="basepath">Base path to use</param>
private static void SetDatItemInfo(DatFile datFile, DatItem datItem, string item, string parent, string basepath)
{
// Get the data to be added as game and item names
string machineName, itemName;
// If the parent is blank, then we have a non-archive file
if (string.IsNullOrWhiteSpace(parent))
{
// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
if (datFile.Header.Type == "SuperDAT")
{
machineName = Path.GetDirectoryName(item.Remove(0, basepath.Length));
itemName = Path.GetFileName(item);
}
// Otherwise, we want just the top level folder as the game, and the file as everything else
else
{
machineName = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0];
itemName = item.Remove(0, (Path.Combine(basepath, machineName).Length));
}
}
// Otherwise, we assume that we have an archive
else
{
// If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
if (datFile.Header.Type == "SuperDAT")
{
machineName = parent;
itemName = datItem.GetName();
}
// Otherwise, we want the archive name as the game, and the file as everything else
else
{
machineName = parent;
itemName = datItem.GetName();
}
}
// Sanitize the names
machineName = machineName.Trim(Path.DirectorySeparatorChar);
itemName = itemName?.Trim(Path.DirectorySeparatorChar) ?? string.Empty;
if (!string.IsNullOrWhiteSpace(machineName) && string.IsNullOrWhiteSpace(itemName))
{
itemName = machineName;
machineName = "Default";
}
// Update machine information
datItem.Machine.Name = machineName;
datItem.Machine.Description = machineName;
// If we have a Disk, then the ".chd" extension needs to be removed
if (datItem.ItemType == ItemType.Disk && itemName.EndsWith(".chd"))
{
itemName = itemName.Substring(0, itemName.Length - 4);
}
// If we have a Media, then the extension needs to be removed
else if (datItem.ItemType == ItemType.Media)
{
if (itemName.EndsWith(".dicf"))
itemName = itemName.Substring(0, itemName.Length - 5);
else if (itemName.EndsWith(".aaru"))
itemName = itemName.Substring(0, itemName.Length - 5);
else if (itemName.EndsWith(".aaruformat"))
itemName = itemName.Substring(0, itemName.Length - 11);
else if (itemName.EndsWith(".aaruf"))
itemName = itemName.Substring(0, itemName.Length - 6);
else if (itemName.EndsWith(".aif"))
itemName = itemName.Substring(0, itemName.Length - 4);
}
// Set the item name back
datItem.SetFields(new Dictionary<Field, string> { [Field.DatItem_Name] = itemName });
}
}
}

3
SabreTools.DatFiles/DatHeader.cs
View File

@@ -7,7 +7,6 @@ using System.Xml.Serialization;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles.Formats;
using SabreTools.IO;
using Newtonsoft.Json;
namespace SabreTools.DatFiles
@@ -1061,7 +1060,7 @@ namespace SabreTools.DatFiles
string filename = string.IsNullOrWhiteSpace(FileName) ? Description : FileName;
// Strip off the extension if it's a holdover from the DAT
if (Parser.HasValidDatExtension(filename))
if (Utilities.HasValidDatExtension(filename))
filename = Path.GetFileNameWithoutExtension(filename);
string outfile = $"{outDir}{filename}{extension}";

504
SabreTools.DatFiles/DatTool.cs
View File

@@ -1,504 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
// TODO: Should each of the individual pieces of partial classes be their own classes?
namespace SabreTools.DatFiles
{
// This file represents all methods related to converting and updating DatFiles
public class DatTool
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Replace item values from the base set represented by the current DAT
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="intDat">DatFile to replace the values in</param>
/// <param name="updateFields">List of Fields representing what should be updated</param>
/// <param name="onlySame">True if descriptions should only be replaced if the game name is the same, false otherwise</param>
public static void BaseReplace(DatFile datFile, DatFile intDat, List<Field> updateFields, bool onlySame)
{
logger.User($"Replacing items in '{intDat.Header.FileName}' from the base DAT");
// If we are matching based on DatItem fields of any sort
if (updateFields.Intersect(DatItem.DatItemFields).Any())
{
// For comparison's sake, we want to use CRC as the base bucketing
datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.Full);
intDat.Items.BucketBy(Field.DatItem_CRC, DedupeType.None);
// Then we do a hashwise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> datItems = intDat.Items[key];
List<DatItem> newDatItems = new List<DatItem>();
foreach (DatItem datItem in datItems)
{
List<DatItem> dupes = datFile.Items.GetDuplicates(datItem, sorted: true);
DatItem newDatItem = datItem.Clone() as DatItem;
// Replace fields from the first duplicate, if we have one
if (dupes.Count > 0)
newDatItem.ReplaceFields(dupes.First(), updateFields);
newDatItems.Add(newDatItem);
}
// Now add the new list to the key
intDat.Items.Remove(key);
intDat.Items.AddRange(key, newDatItems);
});
}
// If we are matching based on Machine fields of any sort
if (updateFields.Intersect(DatItem.MachineFields).Any())
{
// For comparison's sake, we want to use Machine Name as the base bucketing
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.Full);
intDat.Items.BucketBy(Field.Machine_Name, DedupeType.None);
// Then we do a namewise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> datItems = intDat.Items[key];
List<DatItem> newDatItems = new List<DatItem>();
foreach (DatItem datItem in datItems)
{
DatItem newDatItem = datItem.Clone() as DatItem;
if (datFile.Items.ContainsKey(key) && datFile.Items[key].Count() > 0)
newDatItem.Machine.ReplaceFields(datFile.Items[key][0].Machine, updateFields, onlySame);
newDatItems.Add(newDatItem);
}
// Now add the new list to the key
intDat.Items.Remove(key);
intDat.Items.AddRange(key, newDatItems);
});
}
}
/// <summary>
/// Output diffs against a base set represented by the current DAT
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="intDat">DatFile to replace the values in</param>
/// <param name="useGames">True to diff using games, false to use hashes</param>
public static void DiffAgainst(DatFile datFile, DatFile intDat, bool useGames)
{
// For comparison's sake, we want to use a base ordering
if (useGames)
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None);
else
datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.None);
logger.User($"Comparing '{intDat.Header.FileName}' to base DAT");
// For comparison's sake, we want to a the base bucketing
if (useGames)
intDat.Items.BucketBy(Field.Machine_Name, DedupeType.None);
else
intDat.Items.BucketBy(Field.DatItem_CRC, DedupeType.Full);
// Then we compare against the base DAT
List<string> keys = intDat.Items.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Game Against uses game names
if (useGames)
{
// If the base DAT doesn't contain the key, keep it
if (!datFile.Items.ContainsKey(key))
return;
// If the number of items is different, then keep it
if (datFile.Items[key].Count != intDat.Items[key].Count)
return;
// Otherwise, compare by name and hash the remaining files
bool exactMatch = true;
foreach (DatItem item in intDat.Items[key])
{
// TODO: Make this granular to name as well
if (!datFile.Items[key].Contains(item))
{
exactMatch = false;
break;
}
}
// If we have an exact match, remove the game
if (exactMatch)
intDat.Items.Remove(key);
}
// Standard Against uses hashes
else
{
List<DatItem> datItems = intDat.Items[key];
List<DatItem> keepDatItems = new List<DatItem>();
foreach (DatItem datItem in datItems)
{
if (!datFile.Items.HasDuplicates(datItem, true))
keepDatItems.Add(datItem);
}
// Now add the new list to the key
intDat.Items.Remove(key);
intDat.Items.AddRange(key, keepDatItems);
}
});
}
/// <summary>
/// Output cascading diffs
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="datHeaders">Dat headers used optionally</param>
/// <returns>List of DatFiles representing the individually indexed items</returns>
public static List<DatFile> DiffCascade(DatFile datFile, List<DatHeader> datHeaders)
{
// Create a list of DatData objects representing output files
List<DatFile> outDats = new List<DatFile>();
// Ensure the current DatFile is sorted optimally
datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.None);
// Loop through each of the inputs and get or create a new DatData object
InternalStopwatch watch = new InternalStopwatch("Initializing and filling all output DATs");
// Create the DatFiles from the set of headers
DatFile[] outDatsArray = new DatFile[datHeaders.Count];
Parallel.For(0, datHeaders.Count, Globals.ParallelOptions, j =>
{
DatFile diffData = DatFile.Create(datHeaders[j]);
diffData.Items = new ItemDictionary();
FillWithSourceIndex(datFile, diffData, j);
outDatsArray[j] = diffData;
});
outDats = outDatsArray.ToList();
watch.Stop();
return outDats;
}
/// <summary>
/// Output duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffDuplicates(DatFile datFile, List<string> inputs)
{
List<ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return DiffDuplicates(datFile, paths);
}
/// <summary>
/// Output duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffDuplicates(DatFile datFile, List<ParentablePath> inputs)
{
InternalStopwatch watch = new InternalStopwatch("Initializing duplicate DAT");
// Fill in any information not in the base DAT
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
datFile.Header.FileName = "All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Name))
datFile.Header.Name = "datFile.All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Description = "datFile.All DATs";
string post = " (Duplicates)";
DatFile dupeData = DatFile.Create(datFile.Header);
dupeData.Header.FileName += post;
dupeData.Header.Name += post;
dupeData.Header.Description += post;
dupeData.Items = new ItemDictionary();
watch.Stop();
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating duplicate DAT");
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
// Loop through and add the items correctly
foreach (DatItem item in items)
{
if (item.DupeType.HasFlag(DupeType.External))
{
DatItem newrom = item.Clone() as DatItem;
newrom.Machine.Name += $" ({Path.GetFileNameWithoutExtension(inputs[item.Source.Index].CurrentPath)})";
dupeData.Items.Add(key, newrom);
}
}
});
watch.Stop();
return dupeData;
}
/// <summary>
/// Output non-cascading diffs
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static List<DatFile> DiffIndividuals(DatFile datFile, List<string> inputs)
{
List<ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return DiffIndividuals(datFile, paths);
}
/// <summary>
/// Output non-cascading diffs
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static List<DatFile> DiffIndividuals(DatFile datFile, List<ParentablePath> inputs)
{
InternalStopwatch watch = new InternalStopwatch("Initializing all individual DATs");
// Fill in any information not in the base DAT
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
datFile.Header.FileName = "All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Name))
datFile.Header.Name = "All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Description = "All DATs";
// Loop through each of the inputs and get or create a new DatData object
DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
{
string innerpost = $" ({j} - {inputs[j].GetNormalizedFileName(true)} Only)";
DatFile diffData = DatFile.Create(datFile.Header);
diffData.Header.FileName += innerpost;
diffData.Header.Name += innerpost;
diffData.Header.Description += innerpost;
diffData.Items = new ItemDictionary();
outDatsArray[j] = diffData;
});
// Create a list of DatData objects representing individual output files
List<DatFile> outDats = outDatsArray.ToList();
watch.Stop();
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating all individual DATs");
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
// Loop through and add the items correctly
foreach (DatItem item in items)
{
if (item.DupeType.HasFlag(DupeType.Internal) || item.DupeType == 0x00)
outDats[item.Source.Index].Items.Add(key, item);
}
});
watch.Stop();
return outDats.ToList();
}
/// <summary>
/// Output non-duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffNoDuplicates(DatFile datFile, List<string> inputs)
{
List<ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return DiffNoDuplicates(datFile, paths);
}
/// <summary>
/// Output non-duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffNoDuplicates(DatFile datFile, List<ParentablePath> inputs)
{
InternalStopwatch watch = new InternalStopwatch("Initializing no duplicate DAT");
// Fill in any information not in the base DAT
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
datFile.Header.FileName = "All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Name))
datFile.Header.Name = "All DATs";
if (string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Description = "All DATs";
string post = " (No Duplicates)";
DatFile outerDiffData = DatFile.Create(datFile.Header);
outerDiffData.Header.FileName += post;
outerDiffData.Header.Name += post;
outerDiffData.Header.Description += post;
outerDiffData.Items = new ItemDictionary();
watch.Stop();
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating no duplicate DAT");
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
// Loop through and add the items correctly
foreach (DatItem item in items)
{
if (item.DupeType.HasFlag(DupeType.Internal) || item.DupeType == 0x00)
{
DatItem newrom = item.Clone() as DatItem;
newrom.Machine.Name += $" ({Path.GetFileNameWithoutExtension(inputs[item.Source.Index].CurrentPath)})";
outerDiffData.Items.Add(key, newrom);
}
}
});
watch.Stop();
return outerDiffData;
}
/// <summary>
/// Fill a DatFile with all items with a particular source index ID
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="index">Source index ID to retrieve items for</param>
/// <returns>DatFile containing all items with the source index ID/returns>
public static void FillWithSourceIndex(DatFile datFile, DatFile indexDat, int index)
{
// Loop through and add the items for this index to the output
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
foreach (DatItem item in items)
{
if (item.Source.Index == index)
indexDat.Items.Add(key, item);
}
});
}
/// <summary>
/// Populate from multiple paths while returning the invividual headers
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">Paths to DATs to parse</param>
/// <returns>List of DatHeader objects representing headers</returns>
public static List<DatHeader> PopulateUserData(DatFile datFile, List<string> inputs)
{
List<ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return PopulateUserData(datFile, paths);
}
/// <summary>
/// Populate from multiple paths while returning the invividual headers
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">Paths to DATs to parse</param>
/// <returns>List of DatHeader objects representing headers</returns>
public static List<DatHeader> PopulateUserData(DatFile datFile, List<ParentablePath> inputs)
{
DatFile[] datFiles = new DatFile[inputs.Count];
InternalStopwatch watch = new InternalStopwatch("Processing individual DATs");
// Parse all of the DATs into their own DatFiles in the array
Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
{
var input = inputs[i];
logger.User($"Adding DAT: {input.CurrentPath}");
datFiles[i] = DatFile.Create(datFile.Header.CloneFiltering());
Parser.ParseInto(datFiles[i], input, i, keep: true);
});
watch.Stop();
watch.Start("Populating internal DAT");
for (int i = 0; i < inputs.Count; i++)
{
AddFromExisting(datFile, datFiles[i], true);
}
watch.Stop();
return datFiles.Select(d => d.Header).ToList();
}
/// <summary>
/// Add items from another DatFile to the existing DatFile
/// </summary>
/// <param name="addTo">DatFile to add to</param>
/// <param name="addFrom">DatFile to add from</param>
/// <param name="delete">If items should be deleted from the source DatFile</param>
private static void AddFromExisting(DatFile addTo, DatFile addFrom, bool delete = false)
{
// Get the list of keys from the DAT
var keys = addFrom.Items.Keys.ToList();
foreach (string key in keys)
{
// Add everything from the key to the internal DAT
addTo.Items.AddRange(key, addFrom.Items[key]);
// Now remove the key from the source DAT
if (delete)
addFrom.Items.Remove(key);
}
// Now remove the file dictionary from the source DAT
if (delete)
addFrom.Items = null;
}
}
}

380
SabreTools.DatFiles/ItemDictionary.cs
View File

@@ -1,18 +1,13 @@
using System;
using System.Collections;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using System.Xml.Serialization;
using SabreTools.Core;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
using SabreTools.DatFiles.Reports;
using NaturalSort;
using Newtonsoft.Json;
@@ -282,7 +277,7 @@ namespace SabreTools.DatFiles
/// </summary>
/// <remarks>Special count only used by statistics output</remarks>
[JsonIgnore, XmlIgnore]
public long GameCount { get; private set; } = 0;
public long GameCount { get; set; } = 0;
/// <summary>
/// Total uncompressed size
@@ -374,8 +369,6 @@ namespace SabreTools.DatFiles
#endregion
#region Instance methods
#region Accessors
/// <summary>
@@ -462,6 +455,47 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Add statistics from another DatStats object
/// </summary>
/// <param name="stats">DatStats object to add from</param>
public void AddStatistics(ItemDictionary stats)
{
TotalCount += stats.Count;
ArchiveCount += stats.ArchiveCount;
BiosSetCount += stats.BiosSetCount;
ChipCount += stats.ChipCount;
DiskCount += stats.DiskCount;
MediaCount += stats.MediaCount;
ReleaseCount += stats.ReleaseCount;
RomCount += stats.RomCount;
SampleCount += stats.SampleCount;
GameCount += stats.GameCount;
TotalSize += stats.TotalSize;
// Individual hash counts
CRCCount += stats.CRCCount;
MD5Count += stats.MD5Count;
#if NET_FRAMEWORK
RIPEMD160Count += stats.RIPEMD160Count;
#endif
SHA1Count += stats.SHA1Count;
SHA256Count += stats.SHA256Count;
SHA384Count += stats.SHA384Count;
SHA512Count += stats.SHA512Count;
SpamSumCount += stats.SpamSumCount;
// Individual status counts
BaddumpCount += stats.BaddumpCount;
GoodCount += stats.GoodCount;
NodumpCount += stats.NodumpCount;
RemovedCount += stats.RemovedCount;
VerifiedCount += stats.VerifiedCount;
}
/// <summary>
/// Get if the file dictionary contains the key
/// </summary>
@@ -746,47 +780,6 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Add statistics from another DatStats object
/// </summary>
/// <param name="stats">DatStats object to add from</param>
private void AddStatistics(ItemDictionary stats)
{
TotalCount += stats.Count;
ArchiveCount += stats.ArchiveCount;
BiosSetCount += stats.BiosSetCount;
ChipCount += stats.ChipCount;
DiskCount += stats.DiskCount;
MediaCount += stats.MediaCount;
ReleaseCount += stats.ReleaseCount;
RomCount += stats.RomCount;
SampleCount += stats.SampleCount;
GameCount += stats.GameCount;
TotalSize += stats.TotalSize;
// Individual hash counts
CRCCount += stats.CRCCount;
MD5Count += stats.MD5Count;
#if NET_FRAMEWORK
RIPEMD160Count += stats.RIPEMD160Count;
#endif
SHA1Count += stats.SHA1Count;
SHA256Count += stats.SHA256Count;
SHA384Count += stats.SHA384Count;
SHA512Count += stats.SHA512Count;
SpamSumCount += stats.SpamSumCount;
// Individual status counts
BaddumpCount += stats.BaddumpCount;
GoodCount += stats.GoodCount;
NodumpCount += stats.NodumpCount;
RemovedCount += stats.RemovedCount;
VerifiedCount += stats.VerifiedCount;
}
/// <summary>
/// Ensure the key exists in the items dictionary
/// </summary>
@@ -1194,6 +1187,44 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Reset all statistics
/// </summary>
public void ResetStatistics()
{
TotalCount = 0;
ArchiveCount = 0;
BiosSetCount = 0;
ChipCount = 0;
DiskCount = 0;
MediaCount = 0;
ReleaseCount = 0;
RomCount = 0;
SampleCount = 0;
GameCount = 0;
TotalSize = 0;
CRCCount = 0;
MD5Count = 0;
#if NET_FRAMEWORK
RIPEMD160Count = 0;
#endif
SHA1Count = 0;
SHA256Count = 0;
SHA384Count = 0;
SHA512Count = 0;
SpamSumCount = 0;
BaddumpCount = 0;
GoodCount = 0;
NodumpCount = 0;
RemovedCount = 0;
VerifiedCount = 0;
}
/// <summary>
/// Get the highest-order Field value that represents the statistics
/// </summary>
@@ -1230,44 +1261,6 @@ namespace SabreTools.DatFiles
return Field.DatItem_CRC;
}
/// <summary>
/// Reset all statistics
/// </summary>
private void ResetStatistics()
{
TotalCount = 0;
ArchiveCount = 0;
BiosSetCount = 0;
ChipCount = 0;
DiskCount = 0;
MediaCount = 0;
ReleaseCount = 0;
RomCount = 0;
SampleCount = 0;
GameCount = 0;
TotalSize = 0;
CRCCount = 0;
MD5Count = 0;
#if NET_FRAMEWORK
RIPEMD160Count = 0;
#endif
SHA1Count = 0;
SHA256Count = 0;
SHA384Count = 0;
SHA512Count = 0;
SpamSumCount = 0;
BaddumpCount = 0;
GoodCount = 0;
NodumpCount = 0;
RemovedCount = 0;
VerifiedCount = 0;
}
/// <summary>
/// Sort the input DAT and get the key to be used by the item
/// </summary>
@@ -1335,216 +1328,5 @@ namespace SabreTools.DatFiles
}
#endregion
#endregion // Instance methods
#region Static methods
#region Writing
/// <summary>
/// Output the stats for a list of input dats as files in a human-readable format
/// </summary>
/// <param name="inputs">List of input files and folders</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="single">True if single DAT stats are output, false otherwise</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat" > Set the statistics output format to use</param>
public static void OutputStats(
List<string> inputs,
string reportName,
string outDir,
bool single,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
statDatFormat = StatReportFormat.Textfile;
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Get the proper output directory name
outDir = outDir.Ensure();
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outputs = CreateOutStatsNames(outDir, statDatFormat, reportName);
// Make sure we have all files and then order them
List<ParentablePath> files = PathTool.GetFilesOnly(inputs);
files = files
.OrderBy(i => Path.GetDirectoryName(i.CurrentPath))
.ThenBy(i => Path.GetFileName(i.CurrentPath))
.ToList();
// Get all of the writers that we need
List<BaseReport> reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList();
// Write the header, if any
reports.ForEach(report => report.WriteHeader());
// Init all total variables
ItemDictionary totalStats = new ItemDictionary();
// Init directory-level variables
string lastdir = null;
string basepath = null;
ItemDictionary dirStats = new ItemDictionary();
// Now process each of the input files
foreach (ParentablePath file in files)
{
// Get the directory for the current file
string thisdir = Path.GetDirectoryName(file.CurrentPath);
basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath));
// If we don't have the first file and the directory has changed, show the previous directory stats and reset
if (lastdir != null && thisdir != lastdir)
{
// Output separator if needed
reports.ForEach(report => report.WriteMidSeparator());
DatFile lastdirdat = DatFile.Create();
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
}
staticLogger.Verbose($"Beginning stat collection for '{file.CurrentPath}'");
List<string> games = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath);
datdata.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
staticLogger.User($"Adding stats for file '{file.CurrentPath}'\n");
if (single)
{
reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items));
reports.ForEach(report => report.Write());
}
// Add single DAT stats to dir
dirStats.AddStatistics(datdata.Items);
dirStats.GameCount += datdata.Items.Keys.Count();
// Add single DAT stats to totals
totalStats.AddStatistics(datdata.Items);
totalStats.GameCount += datdata.Items.Keys.Count();
// Make sure to assign the new directory
lastdir = thisdir;
}
// Output the directory stats one last time
reports.ForEach(report => report.WriteMidSeparator());
if (single)
{
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
}
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
// Output total DAT stats
reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats));
reports.ForEach(report => report.Write());
// Output footer if needed
reports.ForEach(report => report.WriteFooter());
staticLogger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen");
}
/// <summary>
/// Get the proper extension for the stat output format
/// </summary>
/// <param name="outDir">Output path to use</param>
/// <param name="statDatFormat">StatDatFormat to get the extension for</param>
/// <param name="reportName">Name of the input file to use</param>
/// <returns>Dictionary of output formats mapped to file names</returns>
private static Dictionary<StatReportFormat, string> CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true)
{
Dictionary<StatReportFormat, string> output = new Dictionary<StatReportFormat, string>();
// First try to create the output directory if we need to
if (!Directory.Exists(outDir))
Directory.CreateDirectory(outDir);
// Double check the outDir for the end delim
if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString()))
outDir += Path.DirectorySeparatorChar;
// For each output format, get the appropriate stream writer
output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.Textfile))
output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.CSV))
output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.HTML))
output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.SSV))
output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.TSV))
output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite));
return output;
}
/// <summary>
/// Help generating the outstats name
/// </summary>
/// <param name="outDir">Output directory</param>
/// <param name="extension">Extension to use for the file</param>
/// <param name="reportName">Name of the input file to use</param>
/// <param name="overwrite">True if we ignore existing files, false otherwise</param>
/// <returns>String containing the new filename</returns>
private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite)
{
string outfile = outDir + reportName + extension;
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
if (!overwrite)
{
int i = 1;
while (File.Exists(outfile))
{
outfile = $"{outDir}{reportName}_{i}{extension}";
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
i++;
}
}
return outfile;
}
#endregion
#endregion // Static methods
}
}

1142
SabreTools.DatFiles/Modification.cs
View File

File diff suppressed because it is too large Load Diff

277
SabreTools.DatFiles/Parser.cs
View File

@@ -1,277 +0,0 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using SabreTools.Core;
using SabreTools.IO;
using SabreTools.Logging;
// This file represents all methods related to parsing from a file
namespace SabreTools.DatFiles
{
public class Parser
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Create a DatFile and parse a file into it
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static DatFile CreateAndParse(string filename, bool throwOnError = false)
{
DatFile datFile = DatFile.Create();
ParseInto(datFile, new ParentablePath(filename), throwOnError: throwOnError);
return datFile;
}
/// <summary>
/// Parse a DAT and return all found games and roms within
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="quotes">True if quotes are assumed in supported types (default), false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static void ParseInto(
DatFile datFile,
string filename,
int indexId = 0,
bool keep = false,
bool keepext = false,
bool quotes = true,
bool throwOnError = false)
{
ParentablePath path = new ParentablePath(filename.Trim('"'));
ParseInto(datFile, path, indexId, keep, keepext, quotes, throwOnError);
}
/// <summary>
/// Parse a DAT and return all found games and roms within
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="input">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="quotes">True if quotes are assumed in supported types (default), false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static void ParseInto(
DatFile datFile,
ParentablePath input,
int indexId = 0,
bool keep = false,
bool keepext = false,
bool quotes = true,
bool throwOnError = true)
{
// Get the current path from the filename
string currentPath = input.CurrentPath;
// Check the file extension first as a safeguard
if (!HasValidDatExtension(currentPath))
return;
// If the output filename isn't set already, get the internal filename
datFile.Header.FileName = string.IsNullOrWhiteSpace(datFile.Header.FileName)
? (keepext
? Path.GetFileName(currentPath)
: Path.GetFileNameWithoutExtension(currentPath))
: datFile.Header.FileName;
// If the output type isn't set already, get the internal output type
DatFormat currentPathFormat = GetDatFormat(currentPath);
datFile.Header.DatFormat = datFile.Header.DatFormat == 0 ? currentPathFormat : datFile.Header.DatFormat;
datFile.Items.SetBucketedBy(Field.DatItem_CRC); // Setting this because it can reduce issues later
// Now parse the correct type of DAT
try
{
DatFile.Create(currentPathFormat, datFile, quotes)?.ParseFile(currentPath, indexId, keep, throwOnError);
}
catch (Exception ex)
{
logger.Error(ex, $"Error with file '{currentPath}'");
if (throwOnError) throw ex;
}
}
/// <summary>
/// Get if the given path has a valid DAT extension
/// </summary>
/// <param name="path">Path to check</param>
/// <returns>True if the extension is valid, false otherwise</returns>
public static bool HasValidDatExtension(string path)
{
// Get the extension from the path, if possible
string ext = path.GetNormalizedExtension();
// Check against the list of known DAT extensions
switch (ext)
{
case "csv":
case "dat":
case "json":
case "md5":
case "ripemd160":
case "sfv":
case "sha1":
case "sha256":
case "sha384":
case "sha512":
case "ssv":
case "tsv":
case "txt":
case "xml":
return true;
default:
return false;
}
}
/// <summary>
/// Get what type of DAT the input file is
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <returns>The DatFormat corresponding to the DAT</returns>
private static DatFormat GetDatFormat(string filename)
{
// Limit the output formats based on extension
if (!HasValidDatExtension(filename))
return 0;
// Get the extension from the filename
string ext = filename.GetNormalizedExtension();
// Check if file exists
if (!File.Exists(filename))
return 0;
// Some formats should only require the extension to know
switch (ext)
{
case "csv":
return DatFormat.CSV;
case "json":
return DatFormat.SabreJSON;
case "md5":
return DatFormat.RedumpMD5;
#if NET_FRAMEWORK
case "ripemd160":
return DatFormat.RedumpRIPEMD160;
#endif
case "sfv":
return DatFormat.RedumpSFV;
case "sha1":
return DatFormat.RedumpSHA1;
case "sha256":
return DatFormat.RedumpSHA256;
case "sha384":
return DatFormat.RedumpSHA384;
case "sha512":
return DatFormat.RedumpSHA512;
case "spamsum":
return DatFormat.RedumpSpamSum;
case "ssv":
return DatFormat.SSV;
case "tsv":
return DatFormat.TSV;
}
// For everything else, we need to read it
// Get the first two non-whitespace, non-comment lines to check, if possible
string first = string.Empty, second = string.Empty;
try
{
using (StreamReader sr = File.OpenText(filename))
{
first = sr.ReadLine().ToLowerInvariant();
while ((string.IsNullOrWhiteSpace(first) || first.StartsWith("<!--"))
&& !sr.EndOfStream)
{
first = sr.ReadLine().ToLowerInvariant();
}
if (!sr.EndOfStream)
{
second = sr.ReadLine().ToLowerInvariant();
while (string.IsNullOrWhiteSpace(second) || second.StartsWith("<!--")
&& !sr.EndOfStream)
{
second = sr.ReadLine().ToLowerInvariant();
}
}
}
}
catch { }
// If we have an XML-based DAT
if (first.Contains("<?xml") && first.Contains("?>"))
{
if (second.StartsWith("<!doctype datafile"))
return DatFormat.Logiqx;
else if (second.StartsWith("<!doctype mame")
|| second.StartsWith("<!doctype m1")
|| second.StartsWith("<mame")
|| second.StartsWith("<m1"))
return DatFormat.Listxml;
else if (second.StartsWith("<!doctype softwaredb"))
return DatFormat.OpenMSX;
else if (second.StartsWith("<!doctype softwarelist"))
return DatFormat.SoftwareList;
else if (second.StartsWith("<!doctype sabredat"))
return DatFormat.SabreXML;
else if ((second.StartsWith("<dat") && !second.StartsWith("<datafile"))
|| second.StartsWith("<?xml-stylesheet"))
return DatFormat.OfflineList;
// Older and non-compliant DATs
else
return DatFormat.Logiqx;
}
// If we have an SMDB (SHA-256, Filename, SHA-1, MD5, CRC32)
else if (Regex.IsMatch(first, @"[0-9a-f]{64}\t.*?\t[0-9a-f]{40}\t[0-9a-f]{32}\t[0-9a-f]{8}"))
return DatFormat.EverdriveSMDB;
// If we have an INI-based DAT
else if (first.Contains("[") && first.Contains("]"))
return DatFormat.RomCenter;
// If we have a listroms DAT
else if (first.StartsWith("roms required for driver"))
return DatFormat.Listrom;
// If we have a CMP-based DAT
else if (first.Contains("clrmamepro"))
return DatFormat.ClrMamePro;
else if (first.Contains("romvault"))
return DatFormat.ClrMamePro;
else if (first.Contains("doscenter"))
return DatFormat.DOSCenter;
else if (first.Contains("#Name;Title;Emulator;CloneOf;Year;Manufacturer;Category;Players;Rotation;Control;Status;DisplayCount;DisplayType;AltRomname;AltTitle;Extra"))
return DatFormat.AttractMode;
else
return DatFormat.ClrMamePro;
}
}
}

755
SabreTools.DatFiles/Rebuilder.cs
View File

@@ -1,755 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.IO;
using SabreTools.Logging;
using SabreTools.Skippers;
// This file represents all methods related to rebuilding from a DatFile
namespace SabreTools.DatFiles
{
public class Rebuilder
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Process the DAT and find all matches in input files and folders assuming they're a depot
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildDepot(
DatFile datFile,
List<string> inputs,
string outDir,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return false;
}
// Check that the output directory exists
outDir = outDir.Ensure(create: true);
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
#endregion
bool success = true;
#region Rebuild from depots in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through and get only directories from the input paths
List<string> directories = new List<string>();
Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
logger.Verbose($"Adding depot: {input}");
lock (directories)
{
directories.Add(input);
}
}
});
// If we don't have any directories, we want to exit
if (directories.Count == 0)
return success;
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
datFile.Items.BucketBy(Field.DatItem_SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = datFile.Items.SortedKeys.ToList();
foreach (string hash in keys)
{
// Pre-empt any issues that could arise from string length
if (hash.Length != Constants.SHA1Length)
continue;
logger.User($"Checking hash '{hash}'");
// Get the extension path for the hash
string subpath = Utilities.GetDepotPath(hash, datFile.Header.InputDepot.Depth);
// Find the first depot that includes the hash
string foundpath = null;
foreach (string directory in directories)
{
if (File.Exists(Path.Combine(directory, subpath)))
{
foundpath = Path.Combine(directory, subpath);
break;
}
}
// If we didn't find a path, then we continue
if (foundpath == null)
continue;
// If we have a path, we want to try to get the rom information
GZipArchive archive = new GZipArchive(foundpath);
BaseFile fileinfo = archive.GetTorrentGZFileInfo();
// If the file information is null, then we continue
if (fileinfo == null)
continue;
// Ensure we are sorted correctly (some other calls can change this)
datFile.Items.BucketBy(Field.DatItem_SHA1, DedupeType.None);
// If there are no items in the hash, we continue
if (datFile.Items[hash] == null || datFile.Items[hash].Count == 0)
continue;
// Otherwise, we rebuild that file to all locations that we need to
bool usedInternally;
if (datFile.Items[hash][0].ItemType == ItemType.Disk)
usedInternally = RebuildIndividualFile(datFile, new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
else if (datFile.Items[hash][0].ItemType == ItemType.Media)
usedInternally = RebuildIndividualFile(datFile, new Media(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
else
usedInternally = RebuildIndividualFile(datFile, new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
// If we are supposed to delete the depot file, do so
if (delete && usedInternally)
File.Delete(foundpath);
}
watch.Stop();
#endregion
return success;
}
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildGeneric(
DatFile datFile,
List<string> inputs,
string outDir,
bool quickScan = false,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder,
TreatAsFile asFiles = 0x00)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return false;
}
// Check that the output directory exists
if (!Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
outDir = Path.GetFullPath(outDir);
}
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
#endregion
bool success = true;
#region Rebuild from sources in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through all of the files in all of the inputs
foreach (string input in inputs)
{
// If the input is a file
if (File.Exists(input))
{
logger.User($"Checking file: {input}");
bool rebuilt = RebuildGenericHelper(datFile, input, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
File.Delete(input);
}
// If the input is a directory
else if (Directory.Exists(input))
{
logger.Verbose($"Checking directory: {input}");
foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
{
logger.User($"Checking file: {file}");
bool rebuilt = RebuildGenericHelper(datFile, file, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
File.Delete(input);
}
}
}
watch.Stop();
#endregion
return success;
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
return false;
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List<BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
internalDatItem = new Media(internalFileInfo);
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
internalDatItem = new Disk(internalFileInfo);
else
internalDatItem = new Rom(internalFileInfo);
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return usedExternally || usedInternally;
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool? isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media)
&& !(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba)
&& !(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
datItem = (datItem as Disk).ConvertToRom();
else if (datItem.ItemType == ItemType.Media)
datItem = (datItem as Media).ConvertToRom();
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
return false;
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out List<DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
return true;
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
return true;
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
outputFormat = OutputFormat.Folder;
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
outputFormat = OutputFormat.ParentFolder;
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetFields(new Dictionary<Field, string> { [Field.DatItem_Name] = $"{datItem.GetName()}_{crc}" });
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return rebuilt;
}
/// <summary>
/// Get the rebuild state for a given item
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="stream">Stream representing the input file</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="dupes">Output list of duplicate items to rebuild to</param>
/// <returns>True if the item should be rebuilt, false otherwise</returns>
private static bool ShouldRebuild(DatFile datFile, DatItem datItem, Stream stream, bool inverse, out List<DatItem> dupes)
{
// Find if the file has duplicates in the DAT
dupes = datFile.Items.GetDuplicates(datItem);
bool hasDuplicates = dupes.Count > 0;
// If we have duplicates but we're filtering
if (hasDuplicates && inverse)
{
return false;
}
// If we have duplicates without filtering
else if (hasDuplicates && !inverse)
{
return true;
}
// If we have no duplicates and we're filtering
else if (!hasDuplicates && inverse)
{
string machinename = null;
// Get the item from the current file
Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true));
item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name);
item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name);
// If we are coming from an archive, set the correct machine name
if (machinename != null)
{
item.Machine.Name = machinename;
item.Machine.Description = machinename;
}
dupes.Add(item);
return true;
}
// If we have no duplicates and we're not filtering
else
{
return false;
}
}
/// <summary>
/// Rebuild from TorrentGzip to TorrentGzip
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ, null otherwise</param>
/// <returns>True if rebuilt properly, false otherwise</returns>
private static bool RebuildTorrentGzip(DatFile datFile, DatItem datItem, string file, string outDir, OutputFormat outputFormat, bool? isZip)
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
GZipArchive tgz = new GZipArchive(file);
BaseFile tgzRom = tgz.GetTorrentGZFileInfo();
if (isZip == false && tgzRom != null && (outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba))
{
logger.User($"Matches found for '{Path.GetFileName(datItem.GetName() ?? string.Empty)}', rebuilding accordingly...");
// Get the proper output path
string sha1 = (datItem as Rom).SHA1 ?? string.Empty;
if (outputFormat == OutputFormat.TorrentGzipRomba)
outDir = Path.Combine(outDir, Utilities.GetDepotPath(sha1, datFile.Header.OutputDepot.Depth));
else
outDir = Path.Combine(outDir, sha1 + ".gz");
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outDir));
// Now copy the file over
try
{
File.Copy(file, outDir);
return true;
}
catch
{
return false;
}
}
return false;
}
/// <summary>
/// Rebuild from TorrentXz to TorrentXz
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TXZ, null otherwise</param>
/// <returns>True if rebuilt properly, false otherwise</returns>
private static bool RebuildTorrentXz(DatFile datFile, DatItem datItem, string file, string outDir, OutputFormat outputFormat, bool? isZip)
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
XZArchive txz = new XZArchive(file);
BaseFile txzRom = txz.GetTorrentXZFileInfo();
if (isZip == false && txzRom != null && (outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
logger.User($"Matches found for '{Path.GetFileName(datItem.GetName() ?? string.Empty)}', rebuilding accordingly...");
// Get the proper output path
string sha1 = (datItem as Rom).SHA1 ?? string.Empty;
if (outputFormat == OutputFormat.TorrentXZRomba)
outDir = Path.Combine(outDir, Utilities.GetDepotPath(sha1, datFile.Header.OutputDepot.Depth)).Replace(".gz", ".xz");
else
outDir = Path.Combine(outDir, sha1 + ".xz");
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outDir));
// Now copy the file over
try
{
File.Copy(file, outDir);
return true;
}
catch
{
return false;
}
}
return false;
}
/// <summary>
/// Get the Stream related to a file
/// </summary>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="isZip">Non-null if the input file is an archive</param>
/// <param name="stream">Output stream representing the opened file</param>
/// <returns>True if the stream opening succeeded, false otherwise</returns>
private static bool GetFileStream(DatItem datItem, string file, bool? isZip, out Stream stream)
{
// Get a generic stream for the file
stream = null;
// If we have a zipfile, extract the stream to memory
if (isZip != null)
{
BaseArchive archive = BaseArchive.Create(file);
if (archive != null)
(stream, _) = archive.CopyToStream(datItem.GetName() ?? datItem.ItemType.ToString());
}
// Otherwise, just open the filestream
else
{
stream = File.OpenRead(file);
}
// If the stream is null, then continue
if (stream == null)
return false;
// Seek to the beginning of the stream
if (stream.CanSeek)
stream.Seek(0, SeekOrigin.Begin);
return true;
}
/// <summary>
/// Get the default OutputFormat associated with each PackingFlag
/// </summary>
private static OutputFormat GetOutputFormat(PackingFlag packing)
{
#if NET_FRAMEWORK
switch (packing)
{
case PackingFlag.Zip:
return OutputFormat.TorrentZip;
case PackingFlag.Unzip:
case PackingFlag.Partial:
return OutputFormat.Folder;
case PackingFlag.Flat:
return OutputFormat.ParentFolder;
case PackingFlag.None:
default:
return OutputFormat.Folder;
}
#else
return packing switch
{
PackingFlag.Zip => OutputFormat.TorrentZip,
PackingFlag.Unzip => OutputFormat.Folder,
PackingFlag.Partial => OutputFormat.Folder,
PackingFlag.Flat => OutputFormat.ParentFolder,
PackingFlag.None => OutputFormat.Folder,
_ => OutputFormat.Folder,
};
#endif
}
/// <summary>
/// Get preconfigured Folder for rebuilding
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <returns>Folder configured with proper flags</returns>
private static Folder GetPreconfiguredFolder(DatFile datFile, bool date, OutputFormat outputFormat)
{
Folder outputArchive = Folder.Create(outputFormat);
if (outputArchive is BaseArchive baseArchive && date)
baseArchive.UseDates = date;
// Set the depth fields where appropriate
if (outputArchive is GZipArchive gzipArchive)
gzipArchive.Depth = datFile.Header.OutputDepot.Depth;
else if (outputArchive is XZArchive xzArchive)
xzArchive.Depth = datFile.Header.OutputDepot.Depth;
return outputArchive;
}
/// <summary>
/// Get string value from input OutputFormat
/// </summary>
/// <param name="itemType">OutputFormat to get value from</param>
/// <returns>String value corresponding to the OutputFormat</returns>
private static string FromOutputFormat(OutputFormat itemType)
{
#if NET_FRAMEWORK
switch (itemType)
{
case OutputFormat.Folder:
case OutputFormat.ParentFolder:
return "directory";
case OutputFormat.TapeArchive:
return "TAR";
case OutputFormat.Torrent7Zip:
return "Torrent7Z";
case OutputFormat.TorrentGzip:
case OutputFormat.TorrentGzipRomba:
return "TorrentGZ";
case OutputFormat.TorrentLRZip:
return "TorrentLRZ";
case OutputFormat.TorrentRar:
return "TorrentRAR";
case OutputFormat.TorrentXZ:
case OutputFormat.TorrentXZRomba:
return "TorrentXZ";
case OutputFormat.TorrentZip:
return "TorrentZip";
default:
return null;
}
#else
return itemType switch
{
OutputFormat.Folder => "directory",
OutputFormat.ParentFolder => "directory",
OutputFormat.TapeArchive => "TAR",
OutputFormat.Torrent7Zip => "Torrent7Z",
OutputFormat.TorrentGzip => "TorrentGZ",
OutputFormat.TorrentGzipRomba => "TorrentGZ",
OutputFormat.TorrentLRZip => "TorrentLRZ",
OutputFormat.TorrentRar => "TorrentRAR",
OutputFormat.TorrentXZ => "TorrentXZ",
OutputFormat.TorrentXZRomba => "TorrentXZ",
OutputFormat.TorrentZip => "TorrentZip",
_ => null,
};
#endif
}
}
}

2
SabreTools.DatFiles/Reports/BaseReport.cs
View File

@@ -2,8 +2,8 @@
using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
// TODO: Reports namespace no longer is circular with DatFiles
namespace SabreTools.DatFiles.Reports
{
/// <summary>

4
SabreTools.DatFiles/SabreTools.DatFiles.csproj
View File

@@ -14,16 +14,12 @@
<ItemGroup>
<ProjectReference Include="..\SabreTools.Core\SabreTools.Core.csproj" />
<ProjectReference Include="..\SabreTools.DatItems\SabreTools.DatItems.csproj" />
<ProjectReference Include="..\SabreTools.FileTypes\SabreTools.FileTypes.csproj" />
<ProjectReference Include="..\SabreTools.Filtering\SabreTools.Filtering.csproj" />
<ProjectReference Include="..\SabreTools.IO\SabreTools.IO.csproj" />
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />
<ProjectReference Include="..\SabreTools.Skippers\SabreTools.Skippers.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" />
</ItemGroup>
</Project>

434
SabreTools.DatFiles/Splitter.cs
View File

@@ -1,434 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
using NaturalSort;
// This file represents all methods related to splitting a DatFile into multiple
namespace SabreTools.DatFiles
{
// TODO: Implement Level split
public class Splitter
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Split a DAT by input extensions
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="extA">List of extensions to split on (first DAT)</param>
/// <param name="extB">List of extensions to split on (second DAT)</param>
/// <returns>Extension Set A and Extension Set B DatFiles</returns>
public static (DatFile extADat, DatFile extBDat) SplitByExtension(DatFile datFile, List<string> extA, List<string> extB)
{
// If roms is empty, return false
if (datFile.Items.TotalCount == 0)
return (null, null);
// Make sure all of the extensions don't have a dot at the beginning
var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant());
string newExtAString = string.Join(",", newExtA);
var newExtB = extB.Select(s => s.TrimStart('.').ToLowerInvariant());
string newExtBString = string.Join(",", newExtB);
// Set all of the appropriate outputs for each of the subsets
DatFile extADat = DatFile.Create(datFile.Header.CloneStandard());
extADat.Header.FileName += $" ({newExtAString})";
extADat.Header.Name += $" ({newExtAString})";
extADat.Header.Description += $" ({newExtAString})";
DatFile extBDat = DatFile.Create(datFile.Header.CloneStandard());
extBDat.Header.FileName += $" ({newExtBString})";
extBDat.Header.Name += $" ({newExtBString})";
extBDat.Header.Description += $" ({newExtBString})";
// Now separate the roms accordingly
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
if (newExtA.Contains((item.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extADat.Items.Add(key, item);
}
else if (newExtB.Contains((item.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extBDat.Items.Add(key, item);
}
else
{
extADat.Items.Add(key, item);
extBDat.Items.Add(key, item);
}
}
});
// Then return both DatFiles
return (extADat, extBDat);
}
/// <summary>
/// Split a DAT by best available hashes
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <returns>Dictionary of Field to DatFile mappings</returns>
public static Dictionary<Field, DatFile> SplitByHash(DatFile datFile)
{
// Create each of the respective output DATs
logger.User("Creating and populating new DATs");
// Create the set of field-to-dat mappings
Dictionary<Field, DatFile> fieldDats = new Dictionary<Field, DatFile>();
// TODO: Can this be made into a loop?
fieldDats[Field.DatItem_Status] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_Status].Header.FileName += " (Nodump)";
fieldDats[Field.DatItem_Status].Header.Name += " (Nodump)";
fieldDats[Field.DatItem_Status].Header.Description += " (Nodump)";
fieldDats[Field.DatItem_SHA512] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_SHA512].Header.FileName += " (SHA-512)";
fieldDats[Field.DatItem_SHA512].Header.Name += " (SHA-512)";
fieldDats[Field.DatItem_SHA512].Header.Description += " (SHA-512)";
fieldDats[Field.DatItem_SHA384] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_SHA384].Header.FileName += " (SHA-384)";
fieldDats[Field.DatItem_SHA384].Header.Name += " (SHA-384)";
fieldDats[Field.DatItem_SHA384].Header.Description += " (SHA-384)";
fieldDats[Field.DatItem_SHA256] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_SHA256].Header.FileName += " (SHA-256)";
fieldDats[Field.DatItem_SHA256].Header.Name += " (SHA-256)";
fieldDats[Field.DatItem_SHA256].Header.Description += " (SHA-256)";
fieldDats[Field.DatItem_SHA1] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_SHA1].Header.FileName += " (SHA-1)";
fieldDats[Field.DatItem_SHA1].Header.Name += " (SHA-1)";
fieldDats[Field.DatItem_SHA1].Header.Description += " (SHA-1)";
#if NET_FRAMEWORK
fieldDats[Field.DatItem_RIPEMD160] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_RIPEMD160].Header.FileName += " (RIPEMD160)";
fieldDats[Field.DatItem_RIPEMD160].Header.Name += " (RIPEMD160)";
fieldDats[Field.DatItem_RIPEMD160].Header.Description += " (RIPEMD160)";
#endif
fieldDats[Field.DatItem_MD5] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_MD5].Header.FileName += " (MD5)";
fieldDats[Field.DatItem_MD5].Header.Name += " (MD5)";
fieldDats[Field.DatItem_MD5].Header.Description += " (MD5)";
fieldDats[Field.DatItem_CRC] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.DatItem_CRC].Header.FileName += " (CRC)";
fieldDats[Field.DatItem_CRC].Header.Name += " (CRC)";
fieldDats[Field.DatItem_CRC].Header.Description += " (CRC)";
fieldDats[Field.NULL] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Field.NULL].Header.FileName += " (Other)";
fieldDats[Field.NULL].Header.Name += " (Other)";
fieldDats[Field.NULL].Header.Description += " (Other)";
// Now populate each of the DAT objects in turn
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the file is not a Disk, Media, or Rom, continue
if (item.ItemType != ItemType.Disk && item.ItemType != ItemType.Media && item.ItemType != ItemType.Rom)
return;
// If the file is a nodump
if ((item.ItemType == ItemType.Rom && (item as Rom).ItemStatus == ItemStatus.Nodump)
|| (item.ItemType == ItemType.Disk && (item as Disk).ItemStatus == ItemStatus.Nodump))
{
fieldDats[Field.DatItem_Status].Items.Add(key, item);
}
// If the file has a SHA-512
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA512)))
{
fieldDats[Field.DatItem_SHA512].Items.Add(key, item);
}
// If the file has a SHA-384
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA384)))
{
fieldDats[Field.DatItem_SHA384].Items.Add(key, item);
}
// If the file has a SHA-256
else if ((item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).SHA256))
|| (item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA256)))
{
fieldDats[Field.DatItem_SHA256].Items.Add(key, item);
}
// If the file has a SHA-1
else if ((item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace((item as Disk).SHA1))
|| (item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).SHA1))
|| (item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA1)))
{
fieldDats[Field.DatItem_SHA1].Items.Add(key, item);
}
#if NET_FRAMEWORK
// If the file has a RIPEMD160
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).RIPEMD160)))
{
fieldDats[Field.DatItem_RIPEMD160].Items.Add(key, item);
}
#endif
// If the file has an MD5
else if ((item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace((item as Disk).MD5))
|| (item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).MD5))
|| (item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).MD5)))
{
fieldDats[Field.DatItem_MD5].Items.Add(key, item);
}
// If the file has a CRC
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).CRC)))
{
fieldDats[Field.DatItem_CRC].Items.Add(key, item);
}
else
{
fieldDats[Field.NULL].Items.Add(key, item);
}
}
});
return fieldDats;
}
/// <summary>
/// Split a SuperDAT by lowest available directory level
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="shortname">True if short names should be used, false otherwise</param>
/// <param name="basedat">True if original filenames should be used as the base for output filename, false otherwise</param>
/// <returns>True if split succeeded, false otherwise</returns>
public static bool SplitByLevel(DatFile datFile, string outDir, bool shortname, bool basedat)
{
// First, bucket by games so that we can do the right thing
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None, lower: false, norename: true);
// Create a temporary DAT to add things to
DatFile tempDat = DatFile.Create(datFile.Header);
tempDat.Header.Name = null;
// Sort the input keys
List<string> keys = datFile.Items.Keys.ToList();
keys.Sort(SplitByLevelSort);
// Then, we loop over the games
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Here, the key is the name of the game to be used for comparison
if (tempDat.Header.Name != null && tempDat.Header.Name != Path.GetDirectoryName(key))
{
// Reset the DAT for the next items
tempDat = DatFile.Create(datFile.Header);
tempDat.Header.Name = null;
}
// Clean the input list and set all games to be pathless
List<DatItem> items = datFile.Items[key];
items.ForEach(item => item.Machine.Name = Path.GetFileName(item.Machine.Name));
items.ForEach(item => item.Machine.Description = Path.GetFileName(item.Machine.Description));
// Now add the game to the output DAT
tempDat.Items.AddRange(key, items);
// Then set the DAT name to be the parent directory name
tempDat.Header.Name = Path.GetDirectoryName(key);
});
return true;
}
/// <summary>
/// Helper function for SplitByLevel to sort the input game names
/// </summary>
/// <param name="a">First string to compare</param>
/// <param name="b">Second string to compare</param>
/// <returns>-1 for a coming before b, 0 for a == b, 1 for a coming after b</returns>
private static int SplitByLevelSort(string a, string b)
{
NaturalComparer nc = new NaturalComparer();
int adeep = a.Count(c => c == '/' || c == '\\');
int bdeep = b.Count(c => c == '/' || c == '\\');
if (adeep == bdeep)
return nc.Compare(a, b);
return adeep - bdeep;
}
/// <summary>
/// Helper function for SplitByLevel to clean and write out a DAT
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="newDatFile">DAT to clean and write out</param>
/// <param name="outDir">Directory to write out to</param>
/// <param name="shortname">True if short naming scheme should be used, false otherwise</param>
/// <param name="restore">True if original filenames should be used as the base for output filename, false otherwise</param>
private static void SplitByLevelHelper(DatFile datFile, DatFile newDatFile, string outDir, bool shortname, bool restore)
{
// Get the name from the DAT to use separately
string name = newDatFile.Header.Name;
string expName = name.Replace("/", " - ").Replace("\\", " - ");
// Now set the new output values
newDatFile.Header.FileName = WebUtility.HtmlDecode(string.IsNullOrWhiteSpace(name)
? datFile.Header.FileName
: (shortname
? Path.GetFileName(name)
: expName
)
);
newDatFile.Header.FileName = restore ? $"{datFile.Header.FileName} ({newDatFile.Header.FileName})" : newDatFile.Header.FileName;
newDatFile.Header.Name = $"{datFile.Header.Name} ({expName})";
newDatFile.Header.Description = string.IsNullOrWhiteSpace(datFile.Header.Description) ? newDatFile.Header.Name : $"{datFile.Header.Description} ({expName})";
newDatFile.Header.Type = null;
// Write out the temporary DAT to the proper directory
Writer.Write(newDatFile, outDir);
}
/// <summary>
/// Split a DAT by size of Rom
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="radix">Long value representing the split point</param>
/// <returns>Less Than and Greater Than DatFiles</returns>
public static (DatFile lessThan, DatFile greaterThan) SplitBySize(DatFile datFile, long radix)
{
// Create each of the respective output DATs
logger.User("Creating and populating new DATs");
DatFile lessThan = DatFile.Create(datFile.Header.CloneStandard());
lessThan.Header.FileName += $" (less than {radix})";
lessThan.Header.Name += $" (less than {radix})";
lessThan.Header.Description += $" (less than {radix})";
DatFile greaterThan = DatFile.Create(datFile.Header.CloneStandard());
greaterThan.Header.FileName += $" (equal-greater than {radix})";
greaterThan.Header.Name += $" (equal-greater than {radix})";
greaterThan.Header.Description += $" (equal-greater than {radix})";
// Now populate each of the DAT objects in turn
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the file is not a Rom, it automatically goes in the "lesser" dat
if (item.ItemType != ItemType.Rom)
lessThan.Items.Add(key, item);
// If the file is a Rom and has no size, put it in the "lesser" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size == null)
lessThan.Items.Add(key, item);
// If the file is a Rom and less than the radix, put it in the "lesser" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size < radix)
lessThan.Items.Add(key, item);
// If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size >= radix)
greaterThan.Items.Add(key, item);
}
});
// Then return both DatFiles
return (lessThan, greaterThan);
}
/// <summary>
/// Split a DAT by type of DatItem
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <returns>Dictionary of ItemType to DatFile mappings</returns>
public static Dictionary<ItemType, DatFile> SplitByType(DatFile datFile)
{
// Create each of the respective output DATs
logger.User("Creating and populating new DATs");
// Create the set of type-to-dat mappings
Dictionary<ItemType, DatFile> typeDats = new Dictionary<ItemType, DatFile>();
// We only care about a subset of types
List<ItemType> outputTypes = new List<ItemType>
{
ItemType.Disk,
ItemType.Media,
ItemType.Rom,
ItemType.Sample,
};
// Setup all of the DatFiles
foreach (ItemType itemType in outputTypes)
{
typeDats[itemType] = DatFile.Create(datFile.Header.CloneStandard());
typeDats[itemType].Header.FileName += $" ({itemType})";
typeDats[itemType].Header.Name += $" ({itemType})";
typeDats[itemType].Header.Description += $" ({itemType})";
}
// Now populate each of the DAT objects in turn
Parallel.ForEach(outputTypes, Globals.ParallelOptions, itemType =>
{
FillWithItemType(datFile, typeDats[itemType], itemType);
});
return typeDats;
}
/// <summary>
/// Fill a DatFile with all items with a particular ItemType
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="itemType">ItemType to retrieve items for</param>
/// <returns>DatFile containing all items with the ItemType/returns>
private static void FillWithItemType(DatFile datFile, DatFile indexDat, ItemType itemType)
{
// Loop through and add the items for this index to the output
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
foreach (DatItem item in items)
{
if (item.ItemType == itemType)
indexDat.Items.Add(key, item);
}
});
}
}
}

151
SabreTools.DatFiles/Verification.cs
View File

@@ -1,151 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.Logging;
// This file represents all methods related to verifying with a DatFile
namespace SabreTools.DatFiles
{
public class Verification
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Verify a DatFile against a set of depots, leaving only missing files
/// </summary>
/// <param name="datFile">Current DatFile object to verify against</param>
/// <param name="inputs">List of input directories to compare against</param>
/// <returns>True if verification was a success, false otherwise</returns>
public static bool VerifyDepot(DatFile datFile, List<string> inputs)
{
bool success = true;
InternalStopwatch watch = new InternalStopwatch("Verifying all from supplied depots");
// Now loop through and get only directories from the input paths
List<string> directories = new List<string>();
foreach (string input in inputs)
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
logger.Verbose($"Adding depot: {input}");
directories.Add(input);
}
}
// If we don't have any directories, we want to exit
if (directories.Count == 0)
return success;
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
datFile.Items.BucketBy(Field.DatItem_SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = datFile.Items.SortedKeys.ToList();
foreach (string hash in keys)
{
// Pre-empt any issues that could arise from string length
if (hash.Length != Constants.SHA1Length)
continue;
logger.User($"Checking hash '{hash}'");
// Get the extension path for the hash
string subpath = Utilities.GetDepotPath(hash, datFile.Header.InputDepot.Depth);
// Find the first depot that includes the hash
string foundpath = null;
foreach (string directory in directories)
{
if (File.Exists(Path.Combine(directory, subpath)))
{
foundpath = Path.Combine(directory, subpath);
break;
}
}
// If we didn't find a path, then we continue
if (foundpath == null)
continue;
// If we have a path, we want to try to get the rom information
GZipArchive tgz = new GZipArchive(foundpath);
BaseFile fileinfo = tgz.GetTorrentGZFileInfo();
// If the file information is null, then we continue
if (fileinfo == null)
continue;
// Now we want to remove all duplicates from the DAT
datFile.Items.GetDuplicates(new Rom(fileinfo))
.AddRange(datFile.Items.GetDuplicates(new Disk(fileinfo)));
}
watch.Stop();
// Set fixdat headers in case of writing out
datFile.Header.FileName = $"fixDAT_{datFile.Header.FileName}";
datFile.Header.Name = $"fixDAT_{datFile.Header.Name}";
datFile.Header.Description = $"fixDAT_{datFile.Header.Description}";
datFile.Items.ClearMarked();
return success;
}
/// <summary>
/// Verify a DatFile against a set of inputs, leaving only missing files
/// </summary>
/// <param name="datFile">Current DatFile object to verify against</param>
/// <param name="hashOnly">True if only hashes should be checked, false for full file information</param>
/// <returns>True if verification was a success, false otherwise</returns>
public static bool VerifyGeneric(DatFile datFile, bool hashOnly)
{
bool success = true;
// Force bucketing according to the flags
datFile.Items.SetBucketedBy(Field.NULL);
if (hashOnly)
datFile.Items.BucketBy(Field.DatItem_CRC, DedupeType.Full);
else
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.Full);
// Then mark items for removal
var keys = datFile.Items.SortedKeys.ToList();
foreach (string key in keys)
{
List<DatItem> items = datFile.Items[key];
for (int i = 0; i < items.Count; i++)
{
// Unmatched items will have a source ID of int.MaxValue, remove all others
if (items[i].Source.Index != int.MaxValue)
items[i].Remove = true;
}
// Set the list back, just in case
datFile.Items[key] = items;
}
// Set fixdat headers in case of writing out
datFile.Header.FileName = $"fixDAT_{datFile.Header.FileName}";
datFile.Header.Name = $"fixDAT_{datFile.Header.Name}";
datFile.Header.Description = $"fixDAT_{datFile.Header.Description}";
datFile.Items.ClearMarked();
return success;
}
}
}

172
SabreTools.DatFiles/Writer.cs
View File

@@ -1,172 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles.Reports;
using SabreTools.IO;
using SabreTools.Logging;
// This file represents all methods related to writing to a file
namespace SabreTools.DatFiles
{
public class Writer
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Create and open an output file for writing direct from a dictionary
/// </summary>
/// <param name="datFile">Current DatFile object to write from</param>
/// <param name="outDir">Set the output directory (current directory on null)</param>
/// <param name="overwrite">True if files should be overwritten (default), false if they should be renamed instead</param>
/// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
/// <param name="quotes">True if quotes are assumed in supported types (default), false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DAT was written correctly, false otherwise</returns>
public static bool Write(
DatFile datFile,
string outDir,
bool overwrite = true,
bool ignoreblanks = false,
bool quotes = true,
bool throwOnError = false)
{
// If we have nothing writable, abort
if (!HasWritable(datFile))
{
logger.User("There were no items to write out!");
return false;
}
// Ensure the output directory is set and created
outDir = outDir.Ensure(create: true);
// If the DAT has no output format, default to XML
if (datFile.Header.DatFormat == 0)
{
logger.Verbose("No DAT format defined, defaulting to XML");
datFile.Header.DatFormat = DatFormat.Logiqx;
}
// Make sure that the three essential fields are filled in
EnsureHeaderFields(datFile);
// Bucket roms by game name, if not already
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None);
// Output the number of items we're going to be writing
logger.User($"A total of {datFile.Items.TotalCount - datFile.Items.RemovedCount} items will be written out to '{datFile.Header.FileName}'");
// Get the outfile names
Dictionary<DatFormat, string> outfiles = datFile.Header.CreateOutFileNames(outDir, overwrite);
try
{
// Write out all required formats
Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat =>
{
string outfile = outfiles[datFormat];
try
{
DatFile.Create(datFormat, datFile, quotes)?.WriteToFile(outfile, ignoreblanks, throwOnError);
}
catch (Exception ex)
{
logger.Error(ex, $"Datfile {outfile} could not be written out");
if (throwOnError) throw ex;
}
});
}
catch (Exception ex)
{
logger.Error(ex);
if (throwOnError) throw ex;
return false;
}
return true;
}
/// <summary>
/// Write the stats out to console for the current DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to write from</param>
public static void WriteStatsToConsole(DatFile datFile)
{
if (datFile.Items.RomCount + datFile.Items.DiskCount == 0)
datFile.Items.RecalculateStats();
datFile.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true);
var consoleOutput = BaseReport.Create(StatReportFormat.None, null, true, true);
consoleOutput.ReplaceStatistics(datFile.Header.FileName, datFile.Items.Keys.Count(), datFile.Items);
}
/// <summary>
/// Ensure that FileName, Name, and Description are filled with some value
/// </summary>
/// <param name="datFile">Current DatFile object to write from</param>
private static void EnsureHeaderFields(DatFile datFile)
{
// Empty FileName
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
{
if (string.IsNullOrWhiteSpace(datFile.Header.Name) && string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.FileName = datFile.Header.Name = datFile.Header.Description = "Default";
else if (string.IsNullOrWhiteSpace(datFile.Header.Name) && !string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.FileName = datFile.Header.Name = datFile.Header.Description;
else if (!string.IsNullOrWhiteSpace(datFile.Header.Name) && string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.FileName = datFile.Header.Description = datFile.Header.Name;
else if (!string.IsNullOrWhiteSpace(datFile.Header.Name) && !string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.FileName = datFile.Header.Description;
}
// Filled FileName
else
{
if (string.IsNullOrWhiteSpace(datFile.Header.Name) && string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Name = datFile.Header.Description = datFile.Header.FileName;
else if (string.IsNullOrWhiteSpace(datFile.Header.Name) && !string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Name = datFile.Header.Description;
else if (!string.IsNullOrWhiteSpace(datFile.Header.Name) && string.IsNullOrWhiteSpace(datFile.Header.Description))
datFile.Header.Description = datFile.Header.Name;
}
}
/// <summary>
/// Get if the DatFile has any writable items
/// </summary>
/// <param name="datFile">Current DatFile object to write from</param>
/// <returns>True if there are any writable items, false otherwise</returns>
private static bool HasWritable(DatFile datFile)
{
// Force a statistics recheck, just in case
datFile.Items.RecalculateStats();
// If there's nothing there, abort
if (datFile.Items.TotalCount == 0)
return false;
// If every item is removed, abort
if (datFile.Items.TotalCount == datFile.Items.RemovedCount)
return false;
return true;
}
}
}