claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[OfflineMerge] Remove OfflineMerge

Browse Source 
The features have since been surpassed by the individual diffing
This commit is contained in:
Matt Nadareski
2016-09-12 14:26:18 -07:00
parent 45d68f21bd
commit 30657ac144
6 changed files with 1 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.MD
View File

@@ -41,7 +41,6 @@ The main tool of the SabreTools suite. Performs the majority of the core feature
</li>
<li>Get various statistics on a DAT or folder of DATs including number of roms, disks, and even total size in a human readable format (formerly <i>UncompressedSize</i>; additions requested by Obiwantje)</li>
<li>Filter a DAT by any criteria that the user chooses including wildcard searches (formerly <i>Filter</i>, requested by Obiwantje and others)</li>
<li>Take a current merged DAT, a current missing DAT, and a new merged DAT and get all new files, files to remove, and a new missing DAT. Optionally set all hash values to 0-byte equivalents for rom managers (formerly <i>OfflineMerge</i>, requested by Obiwantje)</li>
<li>Add and remove sources and systems from the database</li>
<li>Retrieve a list of all sources and systems that are available</li>
</ul>

502
SabreTools.Helper/Objects/OfflineMerge.cs
View File

@@ -1,502 +0,0 @@
using SabreTools.Helper;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace SabreTools
{
public class OfflineMerge
{
// Instance variables
private string _currentAllMerged;
private string _currentMissingMerged;
private string _currentNewMerged;
private bool _fake;
private Logger _logger;
/// <summary>
/// Instantiate an OfflineMerge object
/// </summary>
/// <param name="currentAllMerged">Old-current DAT with merged and deduped values</param>
/// <param name="currentMissingMerged">Old-current missing DAT with merged and deduped values</param>
/// <param name="currentNewMerged">New-current DAT with merged and deduped values</param>
/// <param name="fake">True if all values should be replaced with default 0-byte values, false otherwise</param>
/// <param name="logger">Logger object for console and file output</param>
public OfflineMerge (string currentAllMerged, string currentMissingMerged, string currentNewMerged, bool fake, Logger logger)
{
_currentAllMerged = currentAllMerged;
_currentMissingMerged = currentMissingMerged;
_currentNewMerged = currentNewMerged;
_fake = fake;
_logger = logger;
}
/// <summary>
/// Process the supplied inputs and create the four outputs
/// </summary>
/// <returns>True if the files were created properly, false otherwise</returns>
public bool Process()
{
// Check all of the files for validity and break if one doesn't exist
if (_currentAllMerged != "" && !File.Exists(_currentAllMerged))
{
return false;
}
if (_currentMissingMerged != "" && !File.Exists(_currentMissingMerged))
{
return false;
}
if (_currentNewMerged != "" && !File.Exists(_currentNewMerged))
{
return false;
}
// If we have all three DATs, then generate everything
if (_currentAllMerged != "" && _currentMissingMerged != "" && _currentNewMerged != "")
{
// First get the combination Dictionary of currentAllMerged and currentNewMerged
_logger.User("Adding Current and New Merged DATs to the dictionary");
Dat completeDats = new Dat();
completeDats = DatTools.Parse(_currentAllMerged, 0, 0, completeDats, _logger);
completeDats = DatTools.Parse(_currentNewMerged, 0, 0, completeDats, _logger);
// Now get Net New output dictionary [(currentNewMerged)-(currentAllMerged)]
_logger.User("Creating and populating Net New dictionary");
Dictionary<string, List<Rom>> netNew = new Dictionary<string, List<Rom>>();
foreach (string key in completeDats.Files.Keys)
{
List<Rom> templist = RomTools.Merge(completeDats.Files[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
{
if (netNew.ContainsKey(key))
{
netNew[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
netNew.Add(key, temp);
}
}
}
}
// Now create the Unneeded dictionary [(currentAllMerged)-(currentNewMerged)]
_logger.User("Creating and populating Uneeded dictionary");
Dictionary<string, List<Rom>> unneeded = new Dictionary<string, List<Rom>>();
foreach (string key in completeDats.Files.Keys)
{
List<Rom> templist = RomTools.Merge(completeDats.Files[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged)
{
if (unneeded.ContainsKey(key))
{
unneeded[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
unneeded.Add(key, temp);
}
}
}
}
// Now create the New Missing dictionary [(Net New)+(currentMissingMerged-(Unneeded))]
_logger.User("Creating and populating New Missing dictionary");
Dat midMissing = new Dat();
midMissing = DatTools.Parse(_currentMissingMerged, 0, 0, midMissing, _logger);
foreach (string key in unneeded.Keys)
{
if (midMissing.Files.ContainsKey(key))
{
midMissing.Files[key].AddRange(unneeded[key]);
}
else
{
midMissing.Files.Add(key, unneeded[key]);
}
}
Dictionary<string, List<Rom>> newMissing = new Dictionary<string, List<Rom>>();
foreach (string key in midMissing.Files.Keys)
{
List<Rom> templist = RomTools.Merge(midMissing.Files[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentMissingMerged)
{
if (newMissing.ContainsKey(key))
{
newMissing[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
newMissing.Add(key, temp);
}
}
}
}
foreach (string key in netNew.Keys)
{
if (newMissing.ContainsKey(key))
{
newMissing[key].AddRange(netNew[key]);
}
else
{
newMissing.Add(key, netNew[key]);
}
}
// Now create the Have dictionary [(currentNewMerged)-(c)]
_logger.User("Creating and populating Have dictionary");
Dictionary<string, List<Rom>> midHave = new Dictionary<string, List<Rom>>();
foreach (string key in newMissing.Keys)
{
if (midHave.ContainsKey(key))
{
midHave[key].AddRange(newMissing[key]);
}
else
{
midHave.Add(key, newMissing[key]);
}
}
foreach (string key in completeDats.Files.Keys)
{
if (midHave.ContainsKey(key))
{
foreach (Rom rom in completeDats.Files[key])
{
if (rom.Metadata.System == _currentNewMerged)
{
midHave[key].Add(rom);
}
}
}
else
{
List<Rom> roms = new List<Rom>();
foreach (Rom rom in completeDats.Files[key])
{
if (rom.Metadata.System == _currentNewMerged)
{
roms.Add(rom);
}
}
midHave.Add(key, roms);
}
}
Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
foreach (string key in midHave.Keys)
{
List<Rom> templist = RomTools.Merge(midHave[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
{
if (have.ContainsKey(key))
{
have[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
have.Add(key, temp);
}
}
}
}
// If we are supposed to replace everything in the output with default values, do so
if (_fake)
{
_logger.User("Replacing all hashes in Net New with 0-byte values");
List<string> keys = netNew.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = netNew[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
netNew[key] = temp;
}
_logger.User("Replacing all hashes in Unneeded with 0-byte values");
keys = unneeded.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = unneeded[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
unneeded[key] = temp;
}
_logger.User("Replacing all hashes in New Missing with 0-byte values");
keys = newMissing.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = newMissing[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
newMissing[key] = temp;
}
_logger.User("Replacing all hashes in Have with 0-byte values");
keys = have.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = have[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
have[key] = temp;
}
}
// Finally, output all of the files
Dat netNewData = new Dat
{
Name = "Net New",
Description = "Net New",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = netNew,
};
Dat unneededData = new Dat
{
Name = "Unneeded",
Description = "Unneeded",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = unneeded,
};
Dat newMissingData = new Dat
{
Name = "New Missing",
Description = "New Missing",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = newMissing,
};
Dat haveData = new Dat
{
Name = "Have",
Description = "Have",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = have,
};
DatTools.WriteDatfile(netNewData, "", _logger);
DatTools.WriteDatfile(unneededData, "", _logger);
DatTools.WriteDatfile(newMissingData, "", _logger);
DatTools.WriteDatfile(haveData, "", _logger);
return true;
}
// If we only have the old merged and missing, only generate Have
else if (_currentAllMerged != "" && _currentMissingMerged != "")
{
// Now create the Have dictionary [(currentAllMerged)-(currentMissingMerged)]
_logger.User("Creating and populating Have dictionary");
Dat midHave = new Dat();
midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger);
midHave = DatTools.Parse(_currentAllMerged, 0, 0, midHave, _logger);
Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
foreach (string key in midHave.Files.Keys)
{
List<Rom> templist = RomTools.Merge(midHave.Files[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentAllMerged)
{
if (have.ContainsKey(key))
{
have[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
have.Add(key, temp);
}
}
}
}
// If we are supposed to replace everything in the output with default values, do so
if (_fake)
{
_logger.User("Replacing all hashes in Have with 0-byte values");
List<string> keys = have.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = have[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
have[key] = temp;
}
}
Dat haveData = new Dat
{
Name = "Have",
Description = "Have",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = have,
};
DatTools.WriteDatfile(haveData, "", _logger);
return true;
}
// If we only have the new merged and missing, only generate Have
else if (_currentNewMerged != "" && _currentMissingMerged != "")
{
// Now create the Have dictionary [(currentNewMerged)-(currentMissingMerged)]
_logger.User("Creating and populating Have dictionary");
Dat midHave = new Dat();
midHave = DatTools.Parse(_currentMissingMerged, 0, 0, midHave, _logger);
midHave = DatTools.Parse(_currentNewMerged, 0, 0, midHave, _logger);
Dictionary<string, List<Rom>> have = new Dictionary<string, List<Rom>>();
foreach (string key in midHave.Files.Keys)
{
List<Rom> templist = RomTools.Merge(midHave.Files[key], _logger);
foreach (Rom rom in templist)
{
if (rom.Dupe == DupeType.None && rom.Metadata.System == _currentNewMerged)
{
if (have.ContainsKey(key))
{
have[key].Add(rom);
}
else
{
List<Rom> temp = new List<Rom>();
temp.Add(rom);
have.Add(key, temp);
}
}
}
}
// If we are supposed to replace everything in the output with default values, do so
if (_fake)
{
_logger.User("Replacing all hashes in Have with 0-byte values");
List<string> keys = have.Keys.ToList();
foreach (string key in keys)
{
List<Rom> temp = new List<Rom>();
List<Rom> roms = have[key];
for (int i = 0; i < roms.Count; i++)
{
Rom rom = roms[i];
rom.HashData.Size = Constants.SizeZero;
rom.HashData.CRC = Constants.CRCZero;
rom.HashData.MD5 = Constants.MD5Zero;
rom.HashData.SHA1 = Constants.SHA1Zero;
temp.Add(rom);
}
have[key] = temp;
}
}
Dat haveData = new Dat
{
Name = "Have",
Description = "Have",
Version = "",
Date = DateTime.Now.ToString("yyyy-MM-dd"),
Category = "",
Author = "SabreTools",
ForcePacking = ForcePacking.None,
OutputFormat = OutputFormat.Xml,
MergeRoms = true,
Files = have,
};
DatTools.WriteDatfile(haveData, "", _logger);
return true;
}
return false;
}
}
}

28
SabreTools.Helper/README.1ST
View File

@@ -153,7 +153,6 @@ Included within this tool are a few former standalone executables:
- DatSplit: Split a DAT based on 2 different file extensions
- Filter: Filter a DAT based on various user-defined criteria, optionally using wildcards
- HashSplit: Split a DAT based on the best available hash (No-dump, SHA-1, MD5, CRC)
- OfflineMerge: Use merged DATs to create DATs used for managing offline arrays
- MergeDAT: Merge and optionally dedupe an arbitrary number of DAT files
- SingleGame: Trim game and rom names to fit NTFS length standards, optionally merging
all roms to a single game named "!" and forcing unpack
@@ -438,33 +437,6 @@ Options:
-lsy, --list-systems List all systems (id <= name)
List all systems in the database, ordered by the internal ID and mapped to the name
-ol, --offmerge Update DATS for offline arrays
This is a power user tool for dealing with offline arrays, specifically creating
have and miss lists without having to reconnect drives.
This option will output the following DATs based on the inputs as defined below:
(a) Net New = (New Complete)-(Current Complete)
(b) Unneeded - (Current Complete)-(New Complete)
(c) New Missing - (Net New)+(Current Missing-(Unneeded))
(d) Have - (New Complete)-(New Missing)
OR (Complete or NewComplete) - (Missing) if one is missing
-com= Complete current DAT
This is a merged and deduped DAT that includes all DATs that should be checked
-fix= Complete current Missing
This is a merged and deduped DAT that includes all fixdats representing missing
files
-new= New Complete DAT
This is a merged and deduped DAT that includes all current and updated DATs that
should be checked
-fk, --fake Replace all hashes and sizes by the default
For further offline management, this option will replace all hashes in the output
DATs to be their 0-byte equivalents. This allows for file managers to be used
to still deal with the DATs without reconnecting the drives
-rm, --remove Remove a system or source from the database
Remove a system or source to the DAT database so it can no longer be used

1
SabreTools.Helper/SabreTools.Helper.csproj
View File

@@ -93,7 +93,6 @@
<Compile Include="Objects\GenerateTwo.cs" />
<Compile Include="Objects\Import.cs" />
<Compile Include="Objects\ImportTwo.cs" />
<Compile Include="Objects\OfflineMerge.cs" />
<Compile Include="Resources\Resources.de-DE.Designer.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>

17
SabreTools/Partials/SabreTools_Inits.cs
View File

@@ -570,23 +570,6 @@ namespace SabreTools
}
}
/// <summary>
/// Wrap creating an Offline merged DAT set
/// </summary>
/// <param name="currentAllMerged">Old-current DAT with merged and deduped values</param>
/// <param name="currentMissingMerged">Old-current missing DAT with merged and deduped values</param>
/// <param name="currentNewMerged">New-current DAT with merged and deduped values</param>
/// <param name="fake">True if all values should be replaced with default 0-byte values, false otherwise</param>
private static void InitOfflineMerge(string currentAllMerged, string currentMissingMerged, string currentNewMerged, bool fake)
{
OfflineMerge om = new OfflineMerge(currentAllMerged, currentMissingMerged, currentNewMerged, fake, _logger);
bool success = om.Process();
if (!success)
{
_logger.Warning("At least one complete DAT and the fixdat is needed to run!");
}
}
/// <summary>
/// Wrap getting statistics on a DAT or folder of DATs
/// </summary>

28
SabreTools/SabreTools.cs
View File

@@ -89,7 +89,6 @@ namespace SabreTools
dedup = false,
enableGzip = false,
extsplit = false,
fake = false,
forceunpack = false,
generate = false,
genall = false,
@@ -103,7 +102,6 @@ namespace SabreTools
noMD5 = false,
norename = false,
noSHA1 = false,
offlineMerge = false,
old = false,
quotes = false,
rem = false,
@@ -253,10 +251,6 @@ namespace SabreTools
case "--files":
archivesAsFiles = true;
break;
case "-fk":
case "--fake":
fake = true;
break;
case "-g":
case "--generate":
generate = true;
@@ -329,10 +323,6 @@ namespace SabreTools
case "--output-cmp":
outputFormat |= OutputFormat.ClrMamePro;
break;
case "-ol":
case "--offmerge":
offlineMerge = true;
break;
case "-om":
case "--output-miss":
outputFormat |= OutputFormat.MissFile;
@@ -623,7 +613,7 @@ namespace SabreTools
// If more than one switch is enabled, show the help screen
if (!(add ^ datfromdir ^ datfromdirparallel ^ extsplit ^ generate ^ genall ^ hashsplit ^ import ^ listsrc ^ listsys ^
(merge || diffMode != 0 || update || outputFormat != 0 || tsv != null|| trim) ^ offlineMerge ^ rem ^ stats ^ typesplit))
(merge || diffMode != 0 || update || outputFormat != 0 || tsv != null|| trim) ^ rem ^ stats ^ typesplit))
{
_logger.Error("Only one feature switch is allowed at a time");
Build.Help();
@@ -756,22 +746,6 @@ namespace SabreTools
romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir, maxParallelism);
}
// If we want to run Offline merging mode
else if (offlineMerge)
{
if (!(currentAllMerged == "" && currentMissingMerged == "" && currentNewMerged == ""))
{
InitOfflineMerge(currentAllMerged, currentMissingMerged, currentNewMerged, fake);
}
else
{
_logger.User("All inputs were empty! At least one input is required...");
Build.Help();
_logger.Close();
return;
}
}
// If nothing is set, show the help
else
{