claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Have most splits return individual DatFiles

Browse Source
This commit is contained in:
Matt Nadareski
2020-08-28 10:32:17 -07:00
parent 4bf5a835e7
commit 698a6b5e33
3 changed files with 145 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
SabreTools.Library/DatFiles/DatFile.cs
View File

@@ -295,31 +295,6 @@ namespace SabreTools.Library.DatFiles
} }
} }
/// <summary>
/// Fill a DatFile with all items with a particular source index ID
/// </summary>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="index">Source index ID to retrieve items for</param>
/// <returns>DatFile containing all items with the source index ID/returns>
public void FillWithSourceIndex(DatFile indexDat, int index)
{
// Loop through and add the items for this index to the output
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
foreach (DatItem item in items)
{
if (item.Source.Index == index)
indexDat.Items.Add(key, item);
}
});
}
/// <summary> /// <summary>
/// Output diffs against a base set represented by the current DAT /// Output diffs against a base set represented by the current DAT
/// </summary> /// </summary>
@@ -625,6 +600,56 @@ namespace SabreTools.Library.DatFiles
return outerDiffData; return outerDiffData;
} }
/// <summary>
/// Fill a DatFile with all items with a particular ItemType
/// </summary>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="itemType">ItemType to retrieve items for</param>
/// <returns>DatFile containing all items with the ItemType/returns>
public void FillWithItemType(DatFile indexDat, ItemType itemType)
{
// Loop through and add the items for this index to the output
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
foreach (DatItem item in items)
{
if (item.ItemType == itemType)
indexDat.Items.Add(key, item);
}
});
}
/// <summary>
/// Fill a DatFile with all items with a particular source index ID
/// </summary>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="index">Source index ID to retrieve items for</param>
/// <returns>DatFile containing all items with the source index ID/returns>
public void FillWithSourceIndex(DatFile indexDat, int index)
{
// Loop through and add the items for this index to the output
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
{
List<DatItem> items = DatItem.Merge(Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
return;
foreach (DatItem item in items)
{
if (item.Source.Index == index)
indexDat.Items.Add(key, item);
}
});
}
/// <summary> /// <summary>
/// Populate from multiple paths while returning the invividual headers /// Populate from multiple paths while returning the invividual headers
/// </summary> /// </summary>
@@ -1868,6 +1893,7 @@ namespace SabreTools.Library.DatFiles
#endregion #endregion
// TODO: See if any of the helper methods can be broken up a bit more neatly
#region Populate DAT from Directory #region Populate DAT from Directory
/// <summary> /// <summary>
@@ -2223,6 +2249,7 @@ namespace SabreTools.Library.DatFiles
#endregion #endregion
// TODO: See if any of the helper methods can be broken up a bit more neatly
#region Rebuilding and Verifying #region Rebuilding and Verifying
/// <summary> /// <summary>
@@ -3032,15 +3059,14 @@ namespace SabreTools.Library.DatFiles
/// <summary> /// <summary>
/// Split a DAT by input extensions /// Split a DAT by input extensions
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="extA">List of extensions to split on (first DAT)</param> /// <param name="extA">List of extensions to split on (first DAT)</param>
/// <param name="extB">List of extensions to split on (second DAT)</param> /// <param name="extB">List of extensions to split on (second DAT)</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>Extension Set A and Extension Set B DatFiles</returns>
public bool SplitByExtension(string outDir, List<string> extA, List<string> extB) public (DatFile extADat, DatFile extBDat) SplitByExtension(List<string> extA, List<string> extB)
{ {
// If roms is empty, return false // If roms is empty, return false
if (Items.TotalCount == 0) if (Items.TotalCount == 0)
return false; return (null, null);
// Make sure all of the extensions don't have a dot at the beginning // Make sure all of the extensions don't have a dot at the beginning
var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant()); var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant());
@@ -3050,15 +3076,15 @@ namespace SabreTools.Library.DatFiles
string newExtBString = string.Join(",", newExtB); string newExtBString = string.Join(",", newExtB);
// Set all of the appropriate outputs for each of the subsets // Set all of the appropriate outputs for each of the subsets
DatFile datdataA = Create(Header.CloneStandard()); DatFile extADat = Create(Header.CloneStandard());
datdataA.Header.FileName += $" ({newExtAString})"; extADat.Header.FileName += $" ({newExtAString})";
datdataA.Header.Name += $" ({newExtAString})"; extADat.Header.Name += $" ({newExtAString})";
datdataA.Header.Description += $" ({newExtAString})"; extADat.Header.Description += $" ({newExtAString})";
DatFile datdataB = Create(Header.CloneStandard()); DatFile extBDat = Create(Header.CloneStandard());
datdataB.Header.FileName += $" ({newExtBString})"; extBDat.Header.FileName += $" ({newExtBString})";
datdataB.Header.Name += $" ({newExtBString})"; extBDat.Header.Name += $" ({newExtBString})";
datdataB.Header.Description += $" ({newExtBString})"; extBDat.Header.Description += $" ({newExtBString})";
// Now separate the roms accordingly // Now separate the roms accordingly
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
@@ -3068,25 +3094,22 @@ namespace SabreTools.Library.DatFiles
{ {
if (newExtA.Contains(PathExtensions.GetNormalizedExtension(item.Name))) if (newExtA.Contains(PathExtensions.GetNormalizedExtension(item.Name)))
{ {
datdataA.Items.Add(key, item); extADat.Items.Add(key, item);
} }
else if (newExtB.Contains(PathExtensions.GetNormalizedExtension(item.Name))) else if (newExtB.Contains(PathExtensions.GetNormalizedExtension(item.Name)))
{ {
datdataB.Items.Add(key, item); extBDat.Items.Add(key, item);
} }
else else
{ {
datdataA.Items.Add(key, item); extADat.Items.Add(key, item);
datdataB.Items.Add(key, item); extBDat.Items.Add(key, item);
} }
} }
}); });
// Then write out both files // Then return both DatFiles
bool success = datdataA.Write(outDir); return (extADat, extBDat);
success &= datdataB.Write(outDir);
return success;
} }
/// <summary> /// <summary>
@@ -3094,6 +3117,7 @@ namespace SabreTools.Library.DatFiles
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param> /// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>True if split succeeded, false otherwise</returns>
/// TODO: Can this follow the same pattern as type split?
public bool SplitByHash(string outDir) public bool SplitByHash(string outDir)
{ {
// Create each of the respective output DATs // Create each of the respective output DATs
@@ -3333,23 +3357,22 @@ namespace SabreTools.Library.DatFiles
/// <summary> /// <summary>
/// Split a DAT by size of Rom /// Split a DAT by size of Rom
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="radix">Long value representing the split point</param> /// <param name="radix">Long value representing the split point</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>Less Than and Greater Than DatFiles</returns>
public bool SplitBySize(string outDir, long radix) public (DatFile lessThan, DatFile greaterThan) SplitBySize(long radix)
{ {
// Create each of the respective output DATs // Create each of the respective output DATs
Globals.Logger.User("Creating and populating new DATs"); Globals.Logger.User("Creating and populating new DATs");
DatFile lessDat = Create(Header.CloneStandard()); DatFile lessThan = Create(Header.CloneStandard());
lessDat.Header.FileName += $" (less than {radix})"; lessThan.Header.FileName += $" (less than {radix})";
lessDat.Header.Name += $" (less than {radix})"; lessThan.Header.Name += $" (less than {radix})";
lessDat.Header.Description += $" (less than {radix})"; lessThan.Header.Description += $" (less than {radix})";
DatFile greaterEqualDat = Create(Header.CloneStandard()); DatFile greaterThan = Create(Header.CloneStandard());
greaterEqualDat.Header.FileName += $" (equal-greater than {radix})"; greaterThan.Header.FileName += $" (equal-greater than {radix})";
greaterEqualDat.Header.Name += $" (equal-greater than {radix})"; greaterThan.Header.Name += $" (equal-greater than {radix})";
greaterEqualDat.Header.Description += $" (equal-greater than {radix})"; greaterThan.Header.Description += $" (equal-greater than {radix})";
// Now populate each of the DAT objects in turn // Now populate each of the DAT objects in turn
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
@@ -3359,90 +3382,59 @@ namespace SabreTools.Library.DatFiles
{ {
// If the file is not a Rom, it automatically goes in the "lesser" dat // If the file is not a Rom, it automatically goes in the "lesser" dat
if (item.ItemType != ItemType.Rom) if (item.ItemType != ItemType.Rom)
lessDat.Items.Add(key, item); lessThan.Items.Add(key, item);
// If the file is a Rom and less than the radix, put it in the "lesser" dat // If the file is a Rom and less than the radix, put it in the "lesser" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size < radix) else if (item.ItemType == ItemType.Rom && (item as Rom).Size < radix)
lessDat.Items.Add(key, item); lessThan.Items.Add(key, item);
// If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat // If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size >= radix) else if (item.ItemType == ItemType.Rom && (item as Rom).Size >= radix)
greaterEqualDat.Items.Add(key, item); greaterThan.Items.Add(key, item);
} }
}); });
// Now, output all of the files to the output directory // Then return both DatFiles
Globals.Logger.User("DAT information created, outputting new files"); return (lessThan, greaterThan);
bool success = true;
success &= lessDat.Write(outDir);
success &= greaterEqualDat.Write(outDir);
return success;
} }
/// <summary> /// <summary>
/// Split a DAT by type of DatItem /// Split a DAT by type of DatItem
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param> /// <returns>Dictionary of ItemType to DatFile mappings</returns>
/// <returns>True if split succeeded, false otherwise</returns> public Dictionary<ItemType, DatFile> SplitByType()
public bool SplitByType(string outDir)
{ {
// Create each of the respective output DATs // Create each of the respective output DATs
Globals.Logger.User("Creating and populating new DATs"); Globals.Logger.User("Creating and populating new DATs");
DatFile diskdat = Create(Header.CloneStandard()); // Create the set of type-to-dat mappings
diskdat.Header.FileName += " (Disk)"; Dictionary<ItemType, DatFile> typeDats = new Dictionary<ItemType, DatFile>();
diskdat.Header.Name += " (Disk)";
diskdat.Header.Description += " (Disk)";
DatFile mediadat = Create(Header.CloneStandard()); // We only care about a subset of types
mediadat.Header.FileName += " (Media)"; List<ItemType> outputTypes = new List<ItemType>
mediadat.Header.Name += " (Media)"; {
mediadat.Header.Description += " (Media)"; ItemType.Disk,
ItemType.Media,
ItemType.Rom,
ItemType.Sample,
};
DatFile romdat = Create(Header.CloneStandard()); // Setup all of the DatFiles
romdat.Header.FileName += " (Rom)"; foreach (ItemType itemType in outputTypes)
romdat.Header.Name += " (Rom)"; {
romdat.Header.Description += " (Rom)"; typeDats[itemType] = Create(Header.CloneStandard());
typeDats[itemType].Header.FileName += $" ({itemType})";
DatFile sampledat = Create(Header.CloneStandard()); typeDats[itemType].Header.Name += $" ({itemType})";
sampledat.Header.FileName += " (Sample)"; typeDats[itemType].Header.Description += $" ({itemType})";
sampledat.Header.Name += " (Sample)"; }
sampledat.Header.Description += " (Sample)";
// Now populate each of the DAT objects in turn // Now populate each of the DAT objects in turn
Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key => Parallel.ForEach(outputTypes, Globals.ParallelOptions, itemType =>
{ {
List<DatItem> items = Items[key]; FillWithItemType(typeDats[itemType], itemType);
foreach (DatItem item in items)
{
// If the file is a Disk
if (item.ItemType == ItemType.Disk)
diskdat.Items.Add(key, item);
// If the file is a Media
else if (item.ItemType == ItemType.Media)
mediadat.Items.Add(key, item);
// If the file is a Rom
else if (item.ItemType == ItemType.Rom)
romdat.Items.Add(key, item);
// If the file is a Sample
else if (item.ItemType == ItemType.Sample)
sampledat.Items.Add(key, item);
}
}); });
// Now, output all of the files to the output directory return typeDats;
Globals.Logger.User("DAT information created, outputting new files");
bool success = true;
success &= diskdat.Write(outDir);
success &= mediadat.Write(outDir);
success &= romdat.Write(outDir);
success &= sampledat.Write(outDir);
return success;
} }
#endregion #endregion

42
SabreTools/Features/Split.cs
View File

@@ -1,9 +1,12 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks;
using SabreTools.Library.Data; using SabreTools.Library.Data;
using SabreTools.Library.DatFiles; using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Help; using SabreTools.Library.Help;
using SabreTools.Library.IO; using SabreTools.Library.IO;
using SabreTools.Library.Tools;
namespace SabreTools.Features namespace SabreTools.Features
{ {
@@ -61,10 +64,15 @@ namespace SabreTools.Features
// Extension splitting // Extension splitting
if (splittingMode.HasFlag(SplittingMode.Extension)) if (splittingMode.HasFlag(SplittingMode.Extension))
{ {
internalDat.SplitByExtension( (DatFile extADat, DatFile extBDat) = internalDat.SplitByExtension(GetList(features, ExtAListValue), GetList(features, ExtBListValue));
OutputDir,
GetList(features, ExtAListValue), InternalStopwatch watch = new InternalStopwatch("Outputting extension-split DATs");
GetList(features, ExtBListValue));
// Output both possible DatFiles
extADat.Write(OutputDir);
extBDat.Write(OutputDir);
watch.Stop();
} }
// Hash splitting // Hash splitting
@@ -83,11 +91,33 @@ namespace SabreTools.Features
// Size splitting // Size splitting
if (splittingMode.HasFlag(SplittingMode.Size)) if (splittingMode.HasFlag(SplittingMode.Size))
internalDat.SplitBySize(OutputDir, GetInt64(features, RadixInt64Value)); {
(DatFile lessThan, DatFile greaterThan) = internalDat.SplitBySize(GetInt64(features, RadixInt64Value));
InternalStopwatch watch = new InternalStopwatch("Outputting size-split DATs");
// Output both possible DatFiles
lessThan.Write(OutputDir);
greaterThan.Write(OutputDir);
watch.Stop();
}
// Type splitting // Type splitting
if (splittingMode.HasFlag(SplittingMode.Type)) if (splittingMode.HasFlag(SplittingMode.Type))
internalDat.SplitByType(OutputDir); {
Dictionary<ItemType, DatFile> typeDats = internalDat.SplitByType();
InternalStopwatch watch = new InternalStopwatch("Outputting ItemType DATs");
// Loop through each type DatFile
Parallel.ForEach(typeDats.Keys, Globals.ParallelOptions, itemType =>
{
typeDats[itemType].Write(OutputDir);
});
watch.Stop();
}
} }
} }
} }

1
SabreTools/Features/Update.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using SabreTools.Library.Data; using SabreTools.Library.Data;
using SabreTools.Library.DatFiles; using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems; using SabreTools.Library.DatItems;