diff --git a/.gitignore b/.gitignore index 59210b80..7022312e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,7 @@ /SabreTools.Helper/obj/Release /SabreToolsUI/obj /SabreToolsUI/obj/Debug -/SabreToolsUI/obj/Release \ No newline at end of file +/SabreToolsUI/obj/Release +/SimpleSort/obj +/SimpleSort/obj/Debug +/SimpleSort/obj/Release \ No newline at end of file diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index cf946a73..5eaee062 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -225,6 +225,27 @@ Options: -e Detect and remove mode -r Restore header to file based on SHA-1"); break; + case "SimpleSort": + Console.WriteLine(@"SimpleSort - Basic rebuild using a DAT +----------------------------------------- +Usage: SimpleSort [options] [filename|dirname] ... + +Options: + -?, -h, --help Show this help + -dat= Input DAT to rebuild against (REQUIRED) + -out= Output directory + -t=, --temp= Set the temporary directory to use + -7z={0} Set scanning level for 7z archives + -gz={2} Set scanning level for GZip archives + -rar={2} Set scanning level for RAR archives + -zip={0} Set scanning level for ZIP archives + +SimpleSort scanning levels: + 0 Hash archive and contents + 1 Only hash contents + 2 Only hash archive +"); + break; default: Console.Write("This is the default help output"); break; diff --git a/SabreTools.Helper/Data/Enums.cs b/SabreTools.Helper/Data/Enums.cs index 4a546124..c6ac05e4 100644 --- a/SabreTools.Helper/Data/Enums.cs +++ b/SabreTools.Helper/Data/Enums.cs @@ -107,4 +107,14 @@ namespace SabreTools.Helper MissFile, SabreDat, } + + /// + /// Determines the level to scan archives at + /// + public enum ArchiveScanLevel + { + Both = 0, + Internal, + External, + } } diff --git a/SabreTools.Helper/Tools/DatTools.cs b/SabreTools.Helper/Tools/DatTools.cs index 4593d4c7..0b2818e9 100644 --- a/SabreTools.Helper/Tools/DatTools.cs +++ b/SabreTools.Helper/Tools/DatTools.cs @@ -1484,14 +1484,8 @@ namespace SabreTools.Helper { RomData lastrom = outroms[i]; - // If last is a nodump, skip it - if (lastrom.Nodump) - { - continue; - } - // Get the duplicate status - dupefound = RomDuplicate(rom, lastrom); + dupefound = IsDuplicate(rom, lastrom); // If it's a duplicate, skip adding it to the output but add any missing information if (dupefound) @@ -1579,7 +1573,7 @@ namespace SabreTools.Helper /// Rom to use as a base /// DAT to match against /// List of matched RomData objects - public static List ListDuplicates(RomData lastrom, DatData datdata) + public static List GetDuplicates(RomData lastrom, DatData datdata) { List output = new List(); @@ -1594,7 +1588,7 @@ namespace SabreTools.Helper { foreach (RomData rom in roms) { - if (RomDuplicate(rom, lastrom)) + if (IsDuplicate(rom, lastrom)) { output.Add(rom); } @@ -1610,10 +1604,16 @@ namespace SabreTools.Helper /// Rom to check for duplicate status /// Rom to use as a baseline /// True if the roms are duplicates, false otherwise - public static bool RomDuplicate(RomData rom, RomData lastrom) + public static bool IsDuplicate(RomData rom, RomData lastrom) { bool dupefound = false; + // If either is a nodump, it's never a match + if (rom.Nodump || lastrom.Nodump) + { + return dupefound; + } + if (rom.Type == "rom" && lastrom.Type == "rom") { dupefound = ((rom.Size == lastrom.Size) && diff --git a/SabreTools.sln b/SabreTools.sln index 6dad2856..ec5faffa 100644 --- a/SabreTools.sln +++ b/SabreTools.sln @@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreToolsUI", "SabreToolsU EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.Helper", "SabreTools.Helper\SabreTools.Helper.csproj", "{225A1AFD-0890-44E8-B779-7502665C23A5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleSort", "SimpleSort\SimpleSort.csproj", "{7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -48,6 +50,14 @@ Global {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|Any CPU.Build.0 = Release|Any CPU {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|x64.ActiveCfg = Release|x64 {225A1AFD-0890-44E8-B779-7502665C23A5}.Release|x64.Build.0 = Release|x64 + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|x64.ActiveCfg = Debug|x64 + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Debug|x64.Build.0 = Debug|x64 + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|Any CPU.Build.0 = Release|Any CPU + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|x64.ActiveCfg = Release|x64 + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index b0fd3611..0a98f53f 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -477,7 +477,7 @@ namespace SabreTools { currentNewMerged = temparg.Split('=')[1]; } - else if (temparg.StartsWith("-out=") && outdir == "") + else if (temparg.StartsWith("-out=") || temparg.StartsWith("--out=")) { outdir = temparg.Split('=')[1]; } diff --git a/SimpleSort/App.config b/SimpleSort/App.config new file mode 100644 index 00000000..88fa4027 --- /dev/null +++ b/SimpleSort/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/SimpleSort/Properties/AssemblyInfo.cs b/SimpleSort/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..ef79dcbf --- /dev/null +++ b/SimpleSort/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SimpleSort")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SimpleSort")] +[assembly: AssemblyCopyright("Copyright © 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("7668ffa4-19af-4f5d-8463-c7ef5b080fa4")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SimpleSort/SimpleSort.cs b/SimpleSort/SimpleSort.cs new file mode 100644 index 00000000..c9a390fa --- /dev/null +++ b/SimpleSort/SimpleSort.cs @@ -0,0 +1,467 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; + +using SabreTools.Helper; +using DamienG.Security.Cryptography; +using SharpCompress.Archive; +using SharpCompress.Common; +using SharpCompress.Reader; +using SharpCompress.Writer; + +namespace SabreTools +{ + public class SimpleSort + { + // Private instance variables + private DatData _datdata; + private List _inputs; + private string _outdir; + private string _tempdir; + private ArchiveScanLevel _7z; + private ArchiveScanLevel _gz; + private ArchiveScanLevel _rar; + private ArchiveScanLevel _zip; + private Logger _logger; + + /// + /// Create a new SimpleSort object + /// + /// Name of the DAT to compare against + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// Integer representing the archive handling level for 7z + /// Integer representing the archive handling level for GZip + /// Integer representing the archive handling level for RAR + /// Integer representing the archive handling level for Zip + /// Logger object for file and console output + public SimpleSort(DatData datdata, List inputs, string outdir, string tempdir, + int sevenzip, int gz, int rar, int zip, Logger logger) + { + _datdata = datdata; + _inputs = inputs; + _outdir = (outdir == "" ? "Rebuild" : outdir); + _tempdir = (tempdir == "" ? "__TEMP__" : tempdir); + _7z = (ArchiveScanLevel)(sevenzip < 0 || sevenzip > 2 ? 0 : sevenzip); + _gz = (ArchiveScanLevel)(gz < 0 || gz > 2 ? 0 : gz); + _rar = (ArchiveScanLevel)(rar < 0 || rar > 2 ? 0 : rar); + _zip = (ArchiveScanLevel)(zip < 0 || zip > 2 ? 0 : zip); + _logger = logger; + } + + /// + /// Main entry point for the program + /// + /// List of arguments to be parsed + public static void Main(string[] args) + { + // If output is being redirected, don't allow clear screens + if (!Console.IsOutputRedirected) + { + Console.Clear(); + } + + // Perform initial setup and verification + Logger logger = new Logger(true, "simplesort.log"); + logger.Start(); + Remapping.CreateHeaderSkips(); + + // Credits take precidence over all + if ((new List(args)).Contains("--credits")) + { + Build.Credits(); + return; + } + + // If there's no arguments, show help + if (args.Length == 0) + { + Build.Help(); + logger.Close(); + return; + } + + // Output the title + Build.Start("SimpleSort"); + + // Set all default values + bool help = false, + simpleSort = true; + int sevenzip = 0, + gz = 2, + rar = 2, + zip = 0; + string datfile = "", + outdir = "", + tempdir = ""; + List inputs = new List(); + + // Determine which switches are enabled (with values if necessary) + foreach (string arg in args) + { + switch (arg) + { + case "-?": + case "-h": + case "--help": + help = true; + break; + default: + string temparg = arg.Replace("\"", "").Replace("file://", ""); + + if (temparg.StartsWith("-7z=") || temparg.StartsWith("--7z=")) + { + if (!Int32.TryParse(temparg.Split('=')[1], out sevenzip)) + { + sevenzip = 0; + } + } + else if (temparg.StartsWith("-dat=") || temparg.StartsWith("--dat=")) + { + datfile = temparg.Split('=')[1]; + } + else if (temparg.StartsWith("-gz=") || temparg.StartsWith("--gz=")) + { + if (!Int32.TryParse(temparg.Split('=')[1], out gz)) + { + gz = 2; + } + } + else if (temparg.StartsWith("-out=") || temparg.StartsWith("--out=")) + { + outdir = temparg.Split('=')[1]; + } + else if (temparg.StartsWith("-rar=") || temparg.StartsWith("--rar=")) + { + if (!Int32.TryParse(temparg.Split('=')[1], out rar)) + { + rar = 2; + } + } + else if (temparg.StartsWith("-t=") || temparg.StartsWith("--temp=")) + { + tempdir = temparg.Split('=')[1]; + } + else if (temparg.StartsWith("-zip=") || temparg.StartsWith("--zip=")) + { + if (!Int32.TryParse(temparg.Split('=')[1], out zip)) + { + zip = 0; + } + } + else if (File.Exists(temparg) || Directory.Exists(temparg)) + { + inputs.Add(temparg); + } + else + { + logger.Error("Invalid input detected: " + arg); + Console.WriteLine(); + Build.Help(); + logger.Close(); + return; + } + break; + } + } + + // If help is set, show the help screen + if (help) + { + Build.Help(); + logger.Close(); + return; + } + + // If a switch that requires a filename is set and no file is, show the help screen + if (inputs.Count == 0 && (simpleSort)) + { + logger.Error("This feature requires at least one input"); + Build.Help(); + logger.Close(); + return; + } + + // If we are doing a simple sort + if (simpleSort) + { + if (datfile != "") + { + InitSimpleSort(datfile, inputs, outdir, tempdir, sevenzip, gz, rar, zip, logger); + } + else + { + logger.Error("A datfile is required to use this feature"); + Build.Help(); + logger.Close(); + return; + } + } + + // If nothing is set, show the help + else + { + Build.Help(); + } + + logger.Close(); + return; + } + + /// + /// Wrap sorting files using an input DAT + /// + /// Name of the DAT to compare against + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// Integer representing the archive handling level for 7z + /// Integer representing the archive handling level for GZip + /// Integer representing the archive handling level for RAR + /// Integer representing the archive handling level for Zip + /// Logger object for file and console output + private static void InitSimpleSort(string datfile, List inputs, string outdir, string tempdir, + int sevenzip, int gz, int rar, int zip, Logger logger) + { + DatData datdata = new DatData(); + datdata = DatTools.Parse(datfile, 0, 0, datdata, logger); + + SimpleSort ss = new SimpleSort(datdata, inputs, outdir, tempdir, sevenzip, gz, rar, zip, logger); + ss.Process(); + } + + /// + /// Process the DAT and find all matches in input files and folders + /// + /// + public bool Process() + { + bool success = true; + + // First, check that the output directory exists + if (!Directory.Exists(_outdir)) + { + Directory.CreateDirectory(_outdir); + _outdir = Path.GetFullPath(_outdir); + } + + // Then, loop through and check each of the inputs + _logger.User("Starting to loop through inputs"); + foreach (string input in _inputs) + { + if (File.Exists(input)) + { + _logger.Log("File found: '" + input + "'"); + success &= ProcessFile(input); + CleanTempDirectory(); + } + else if (Directory.Exists(input)) + { + _logger.Log("Directory found: '" + input + "'"); + foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) + { + _logger.Log("File found: '" + file + "'"); + success &= ProcessFile(file); + CleanTempDirectory(); + } + } + else + { + _logger.Error("'" + input + "' is not a file or directory!"); + } + } + + // Now one final delete of the temp directory + Directory.Delete(_tempdir, true); + + return success; + } + + /// + /// Process an individual file against the DAT + /// + /// The name of the input file + /// True if this is in a recurse step and the file should be deleted, false otherwise (default) + /// True if it was processed properly, false otherwise + private bool ProcessFile(string input, bool recurse = false) + { + bool success = true; + + // Get the full path of the input for movement purposes + input = Path.GetFullPath(input); + _logger.User("Beginning processing of '" + input + "'"); + + // Get the hash of the file first + RomData rom = new RomData + { + Name = Path.GetFileName(input), + Type = "rom", + Size = (new FileInfo(input)).Length, + CRC= string.Empty, + MD5 = string.Empty, + SHA1 = string.Empty, + }; + + try + { + Crc32 crc = new Crc32(); + MD5 md5 = MD5.Create(); + SHA1 sha1 = SHA1.Create(); + + using (FileStream fs = File.Open(input, FileMode.Open)) + { + foreach (byte b in crc.ComputeHash(fs)) + { + rom.CRC += b.ToString("x2").ToLower(); + } + } + using (FileStream fs = File.Open(input, FileMode.Open)) + { + rom.MD5 = BitConverter.ToString(md5.ComputeHash(fs)).Replace("-", "").ToLowerInvariant(); ; + } + using (FileStream fs = File.Open(input, FileMode.Open)) + { + rom.SHA1 = BitConverter.ToString(sha1.ComputeHash(fs)).Replace("-", "").ToLowerInvariant(); + } + } + catch (IOException) + { + return false; + } + + // Try to find the matches to the file that was found + List foundroms = DatTools.GetDuplicates(rom, _datdata); + _logger.User("File '" + input + "' had " + foundroms.Count + " matches in the DAT!"); + foreach (RomData found in foundroms) + { + IWriter outarchive = null; + FileStream fs = null; + try + { + fs = File.OpenWrite(_outdir + Path.DirectorySeparatorChar + found.Game + ".zip"); + outarchive = WriterFactory.Open(fs, ArchiveType.Zip, CompressionType.Deflate); + outarchive.Write(found.Name, input); + } + catch + { + if (!File.Exists(_outdir + Path.DirectorySeparatorChar + found.Game + Path.DirectorySeparatorChar + rom.Name)) + { + File.Copy(input, _outdir + Path.DirectorySeparatorChar + found.Game + Path.DirectorySeparatorChar + rom.Name); + } + } + finally + { + outarchive?.Dispose(); + fs?.Close(); + fs?.Dispose(); + } + } + + // Now, if the file is a supported archive type, also run on all files within + bool encounteredErrors = true; + IArchive archive = null; + try + { + archive = ArchiveFactory.Open(input); + ArchiveType at = archive.Type; + _logger.Log("Found archive of type: " + at); + + if ((at == ArchiveType.Zip && _zip != ArchiveScanLevel.External) || + (at == ArchiveType.SevenZip && _7z != ArchiveScanLevel.External) || + (at == ArchiveType.Rar && _rar != ArchiveScanLevel.External)) + { + // Create the temp directory + DirectoryInfo di = Directory.CreateDirectory(_tempdir); + + // Extract all files to the temp directory + IReader reader = archive.ExtractAllEntries(); + reader.WriteAllToDirectory(_tempdir, ExtractOptions.ExtractFullPath); + encounteredErrors = false; + } + else if (at == ArchiveType.GZip && _gz != ArchiveScanLevel.External) + { + // Close the original archive handle + archive.Dispose(); + + // Create the temp directory + DirectoryInfo di = Directory.CreateDirectory(_tempdir); + + using (FileStream itemstream = File.OpenRead(input)) + { + using (FileStream outstream = File.Create(_tempdir + Path.GetFileNameWithoutExtension(input))) + { + using (GZipStream gz = new GZipStream(itemstream, CompressionMode.Decompress)) + { + gz.CopyTo(outstream); + } + } + } + encounteredErrors = false; + } + archive.Dispose(); + } + catch (InvalidOperationException) + { + encounteredErrors = true; + if (archive != null) + { + archive.Dispose(); + } + } + catch (Exception ex) + { + _logger.Error(ex.ToString()); + encounteredErrors = true; + if (archive != null) + { + archive.Dispose(); + } + } + + // Remove the current file if we are in recursion so it's not picked up in the next step + if (recurse) + { + File.Delete(input); + } + + // If no errors were encountered, we loop through the temp directory + if (!encounteredErrors) + { + _logger.User("Archive found! Successfully extracted"); + foreach (string file in Directory.EnumerateFiles(_tempdir, "*", SearchOption.AllDirectories)) + { + success &= ProcessFile(file, true); + } + } + + return success; + } + + /// + /// Cleans out the temporary directory + /// + private void CleanTempDirectory() + { + foreach (string file in Directory.EnumerateFiles(_tempdir, "*", SearchOption.TopDirectoryOnly)) + { + try + { + File.Delete(file); + } + catch { } + } + foreach (string dir in Directory.EnumerateDirectories(_tempdir, "*", SearchOption.TopDirectoryOnly)) + { + try + { + Directory.Delete(dir, true); + } + catch { } + } + } + } +} diff --git a/SimpleSort/SimpleSort.csproj b/SimpleSort/SimpleSort.csproj new file mode 100644 index 00000000..d9d94588 --- /dev/null +++ b/SimpleSort/SimpleSort.csproj @@ -0,0 +1,91 @@ + + + + + Debug + AnyCPU + {7668FFA4-19AF-4F5D-8463-C7EF5B080FA4} + Exe + Properties + SabreTools + SimpleSort + v4.5.2 + 512 + true + + + AnyCPU + true + full + false + ..\..\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + ..\..\Release\ + TRACE + prompt + 4 + + + true + ..\..\Debug-x64\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\Release-x64\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + ..\packages\sharpcompress.0.11.6\lib\net40\SharpCompress.dll + True + + + + + + + + + + + + + + + + + + + + + {225a1afd-0890-44e8-b779-7502665c23a5} + SabreTools.Helper + + + + + \ No newline at end of file diff --git a/SimpleSort/packages.config b/SimpleSort/packages.config new file mode 100644 index 00000000..b19639c6 --- /dev/null +++ b/SimpleSort/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file