claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ALL] Add game dedupe

Browse Source
This commit is contained in:
Matt Nadareski
2017-08-29 11:46:01 -07:00
parent 66524bf034
commit e7683ec0fb
14 changed files with 93 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
SabreTools.Library/Dats/Partials/DatFile.ConvertUpdate.cs
View File

@@ -119,7 +119,7 @@ namespace SabreTools.Library.Dats
datHeaders[i] = new DatFile
{
DatFormat = (DatFormat != 0 ? DatFormat : 0),
MergeRoms = MergeRoms,
DedupeRoms = DedupeRoms,
};
datHeaders[i].Parse(input.Split('¬')[0], i, 0, splitType, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
@@ -187,7 +187,7 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Populating base DAT complete in {0}", DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// For comparison's sake, we want to use CRC as the base ordering
BucketBy(SortedBy.CRC, true);
BucketBy(SortedBy.CRC, DedupeType.Full);
// Now we want to compare each input DAT against the base
List<string> inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true);
@@ -203,7 +203,7 @@ namespace SabreTools.Library.Dats
intDat.Parse(splitpath[0], 1, 1, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
// For comparison's sake, we want to use CRC as the base ordering
intDat.BucketBy(SortedBy.CRC, true);
intDat.BucketBy(SortedBy.CRC, DedupeType.Full);
// Then we do a hashwise comparison against the base DAT
List<string> keys = intDat.Keys.ToList();

28
SabreTools.Library/Dats/Partials/DatFile.Manipulate.cs
View File

@@ -25,10 +25,10 @@ namespace SabreTools.Library.Dats
/// Take the arbitrarily sorted Files Dictionary and convert to one sorted by a user-defined method
/// </summary>
/// <param name="bucketBy">SortedBy enum representing how to sort the individual items</param>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
/// <param name="deduperoms">Dedupe type that should be used</param>
/// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
public void BucketBy(SortedBy bucketBy, bool mergeroms, bool lower = true, bool norename = true)
public void BucketBy(SortedBy bucketBy, DedupeType deduperoms, bool lower = true, bool norename = true)
{
// If we already have the right sorting, trust it
if (_sortedBy == bucketBy)
@@ -48,7 +48,7 @@ namespace SabreTools.Library.Dats
// Create the temporary dictionary to sort into
SortedDictionary<string, List<DatItem>> sortable = new SortedDictionary<string, List<DatItem>>();
Globals.Logger.User("Organizing roms by {0}" + (mergeroms ? " and merging" : ""), bucketBy);
Globals.Logger.User("Organizing roms by {0}" + (deduperoms != DedupeType.None ? " and merging" : ""), bucketBy);
// First do the initial sort of all of the roms
List<string> keys = Keys.ToList();
@@ -152,7 +152,7 @@ namespace SabreTools.Library.Dats
DatItem.Sort(ref sortedlist, false);
// If we're merging the roms, do so
if (mergeroms)
if (deduperoms == DedupeType.Full || (deduperoms == DedupeType.Game && bucketBy == SortedBy.Game))
{
sortedlist = DatItem.Merge(sortedlist);
}
@@ -384,8 +384,8 @@ namespace SabreTools.Library.Dats
/// <summary>
/// Use cdevice_ref tags to get full non-merged sets and remove parenting tags
/// </summary>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
public void CreateDeviceNonMergedSets(bool mergeroms)
/// <param name="mergeroms">Dedupe type to be used</param>
public void CreateDeviceNonMergedSets(DedupeType mergeroms)
{
Globals.Logger.User("Creating device non-merged sets from the DAT");
@@ -406,8 +406,8 @@ namespace SabreTools.Library.Dats
/// <summary>
/// Use cloneof tags to create non-merged sets and remove the tags plus using the device_ref tags to get full sets
/// </summary>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
public void CreateFullyNonMergedSets(bool mergeroms)
/// <param name="mergeroms">Dedupe type to be used</param>
public void CreateFullyNonMergedSets(DedupeType mergeroms)
{
Globals.Logger.User("Creating fully non-merged sets from the DAT");
@@ -432,8 +432,8 @@ namespace SabreTools.Library.Dats
/// <summary>
/// Use cloneof tags to create merged sets and remove the tags
/// </summary>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
public void CreateMergedSets(bool mergeroms)
/// <param name="mergeroms">Dedupe type to be used</param>
public void CreateMergedSets(DedupeType mergeroms)
{
Globals.Logger.User("Creating merged sets from the DAT");
@@ -454,8 +454,8 @@ namespace SabreTools.Library.Dats
/// <summary>
/// Use cloneof tags to create non-merged sets and remove the tags
/// </summary>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
public void CreateNonMergedSets(bool mergeroms)
/// <param name="mergeroms">Dedupe type to be used</param>
public void CreateNonMergedSets(DedupeType mergeroms)
{
Globals.Logger.User("Creating non-merged sets from the DAT");
@@ -476,8 +476,8 @@ namespace SabreTools.Library.Dats
/// <summary>
/// Use cloneof and romof tags to create split sets and remove the tags
/// </summary>
/// <param name="mergeroms">True if roms should be deduped, false otherwise</param>
public void CreateSplitSets(bool mergeroms)
/// <param name="mergeroms">Dedupe type to be used</param>
public void CreateSplitSets(DedupeType mergeroms)
{
Globals.Logger.User("Creating split sets from the DAT");

6
SabreTools.Library/Dats/Partials/DatFile.Rebuild.cs
View File

@@ -148,7 +148,7 @@ namespace SabreTools.Library.Dats
}
// Now that we have a list of depots, we want to sort the input DAT by SHA-1
BucketBy(SortedBy.SHA1, false /* mergeroms */);
BucketBy(SortedBy.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
List<string> hashes = Keys.ToList();
@@ -884,7 +884,7 @@ namespace SabreTools.Library.Dats
}
// Now that we have a list of depots, we want to sort the input DAT by SHA-1
BucketBy(SortedBy.SHA1, false /* mergeroms */);
BucketBy(SortedBy.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
List<string> hashes = Keys.ToList();
@@ -994,7 +994,7 @@ namespace SabreTools.Library.Dats
if (hashOnly)
{
// First we need to sort by hash to get duplicates
BucketBy(SortedBy.SHA1, false /* mergeroms */);
BucketBy(SortedBy.SHA1, DedupeType.None);
// Then follow the same tactics as before
foreach (string key in Keys)

24
SabreTools.Library/Dats/Partials/DatFile.Splitters.cs
View File

@@ -155,7 +155,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile sha512 = new DatFile
{
@@ -176,7 +176,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile sha384 = new DatFile
{
@@ -197,7 +197,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile sha256 = new DatFile
{
@@ -218,7 +218,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile sha1 = new DatFile
{
@@ -239,7 +239,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile md5 = new DatFile
{
@@ -260,7 +260,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile crc = new DatFile
{
@@ -281,7 +281,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile other = new DatFile
{
@@ -302,7 +302,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
// Now populate each of the DAT objects in turn
@@ -405,7 +405,7 @@ namespace SabreTools.Library.Dats
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
// First, organize by games so that we can do the right thing
BucketBy(SortedBy.Game, false /* mergeroms */, lower: false, norename: true);
BucketBy(SortedBy.Game, DedupeType.None, lower: false, norename: true);
// Create a temporary DAT to add things to
DatFile tempDat = new DatFile(this)
@@ -537,7 +537,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile diskdat = new DatFile
{
@@ -558,7 +558,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
DatFile sampledat = new DatFile
{
@@ -579,7 +579,7 @@ namespace SabreTools.Library.Dats
ForceNodump = this.ForceNodump,
ForcePacking = this.ForcePacking,
DatFormat = this.DatFormat,
MergeRoms = this.MergeRoms,
DedupeRoms = this.DedupeRoms,
};
// Now populate each of the DAT objects in turn

4
SabreTools.Library/Dats/Partials/DatFile.Statistics.cs
View File

@@ -205,7 +205,7 @@ namespace SabreTools.Library.Dats
RecalculateStats();
}
BucketBy(SortedBy.Game, false /* mergeroms */, norename: true);
BucketBy(SortedBy.Game, DedupeType.None, norename: true);
if (_totalSize < 0)
{
_totalSize = Int64.MaxValue + _totalSize;
@@ -502,7 +502,7 @@ namespace SabreTools.Library.Dats
List<string> games = new List<string>();
DatFile datdata = new DatFile();
datdata.Parse(filename.Item1, 0, 0);
datdata.BucketBy(SortedBy.Game, false /* mergeroms */, norename: true);
datdata.BucketBy(SortedBy.Game, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
Globals.Logger.User("Adding stats for file '{0}'\n", false, filename.Item1);

14
SabreTools.Library/Dats/Partials/DatFile.Writers.cs
View File

@@ -105,14 +105,18 @@ namespace SabreTools.Library.Dats
recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true);
}
// First bucket by CRC to dedupe if required
if (MergeRoms)
// Bucket and dedupe according to the flag
if (DedupeRoms == DedupeType.Full)
{
BucketBy(SortedBy.CRC, MergeRoms, norename: norename);
BucketBy(SortedBy.CRC, DedupeRoms, norename: norename);
}
else if (DedupeRoms == DedupeType.Game)
{
BucketBy(SortedBy.Game, DedupeRoms, norename: norename);
}
// Bucket roms by game name
BucketBy(SortedBy.Game, false /* mergeRoms */, norename: norename);
// Bucket roms by game name, if not already
BucketBy(SortedBy.Game, DedupeType.None, norename: norename);
// Output the number of items we're going to be writing
Globals.Logger.User("A total of {0} items will be written out to '{1}'", Count, FileName);