diff --git a/.gitignore b/.gitignore index 90df7a71..1a5de6dc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ /SabreTools.DatFiles/obj/ /SabreTools.DatItems/bin/ /SabreTools.DatItems/obj/ +/SabreTools.DatTools/bin/ +/SabreTools.DatTools/obj/ /SabreTools.FileTypes/bin/ /SabreTools.FileTypes/obj/ /SabreTools.Filtering/bin/ diff --git a/RombaSharp/Features/Archive.cs b/RombaSharp/Features/Archive.cs index c7017be5..1ea99192 100644 --- a/RombaSharp/Features/Archive.cs +++ b/RombaSharp/Features/Archive.cs @@ -5,6 +5,7 @@ using System.Linq; using SabreTools.Core; using SabreTools.DatFiles; using SabreTools.DatItems; +using SabreTools.DatTools; using SabreTools.Help; using Microsoft.Data.Sqlite; diff --git a/RombaSharp/Features/BaseFeature.cs b/RombaSharp/Features/BaseFeature.cs index ba1261ea..598e4ef9 100644 --- a/RombaSharp/Features/BaseFeature.cs +++ b/RombaSharp/Features/BaseFeature.cs @@ -9,6 +9,7 @@ using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatFiles; using SabreTools.DatItems; +using SabreTools.DatTools; using SabreTools.FileTypes; using SabreTools.Help; using SabreTools.Logging; diff --git a/RombaSharp/Features/Build.cs b/RombaSharp/Features/Build.cs index 17bbbf01..3d3924b9 100644 --- a/RombaSharp/Features/Build.cs +++ b/RombaSharp/Features/Build.cs @@ -4,6 +4,7 @@ using System.Linq; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; diff --git a/RombaSharp/Features/DatStats.cs b/RombaSharp/Features/DatStats.cs index 8afe98ab..62bbfb27 100644 --- a/RombaSharp/Features/DatStats.cs +++ b/RombaSharp/Features/DatStats.cs @@ -2,7 +2,7 @@ using System.IO; using SabreTools.Core; -using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; namespace RombaSharp.Features @@ -30,7 +30,7 @@ namespace RombaSharp.Features Inputs = new List { Path.GetFullPath(_dats) }; // Now output the stats for all inputs - ItemDictionary.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile); + Statistics.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile); } } } diff --git a/RombaSharp/Features/Diffdat.cs b/RombaSharp/Features/Diffdat.cs index 2abde148..5c8d369f 100644 --- a/RombaSharp/Features/Diffdat.cs +++ b/RombaSharp/Features/Diffdat.cs @@ -2,6 +2,7 @@ using System.IO; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; diff --git a/RombaSharp/Features/Dir2Dat.cs b/RombaSharp/Features/Dir2Dat.cs index a454fa62..9ed60c26 100644 --- a/RombaSharp/Features/Dir2Dat.cs +++ b/RombaSharp/Features/Dir2Dat.cs @@ -4,6 +4,7 @@ using System.IO; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Filtering; using SabreTools.Help; using SabreTools.IO; diff --git a/RombaSharp/Features/EDiffdat.cs b/RombaSharp/Features/EDiffdat.cs index 3ea743ab..e945c6ba 100644 --- a/RombaSharp/Features/EDiffdat.cs +++ b/RombaSharp/Features/EDiffdat.cs @@ -2,6 +2,7 @@ using System.IO; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; diff --git a/RombaSharp/Features/Miss.cs b/RombaSharp/Features/Miss.cs index 5bbc43c0..b7c1398f 100644 --- a/RombaSharp/Features/Miss.cs +++ b/RombaSharp/Features/Miss.cs @@ -3,6 +3,7 @@ using System.IO; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; diff --git a/RombaSharp/Features/RefreshDats.cs b/RombaSharp/Features/RefreshDats.cs index e3ad8eab..4b0dba82 100644 --- a/RombaSharp/Features/RefreshDats.cs +++ b/RombaSharp/Features/RefreshDats.cs @@ -4,6 +4,7 @@ using System.IO; using SabreTools.Core; using SabreTools.DatFiles; using SabreTools.DatItems; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.Logging; using Microsoft.Data.Sqlite; diff --git a/RombaSharp/Features/RescanDepots.cs b/RombaSharp/Features/RescanDepots.cs index 584a4640..2d1e762f 100644 --- a/RombaSharp/Features/RescanDepots.cs +++ b/RombaSharp/Features/RescanDepots.cs @@ -4,6 +4,7 @@ using System.IO; using SabreTools.Core; using SabreTools.DatFiles; using SabreTools.DatItems; +using SabreTools.DatTools; using SabreTools.Help; using Microsoft.Data.Sqlite; diff --git a/RombaSharp/RombaSharp.csproj b/RombaSharp/RombaSharp.csproj index 2b4b0e7c..0ae41ecf 100644 --- a/RombaSharp/RombaSharp.csproj +++ b/RombaSharp/RombaSharp.csproj @@ -19,6 +19,7 @@ + diff --git a/SabreTools.Core/Tools/Utilities.cs b/SabreTools.Core/Tools/Utilities.cs index 099db57c..1e9d0640 100644 --- a/SabreTools.Core/Tools/Utilities.cs +++ b/SabreTools.Core/Tools/Utilities.cs @@ -146,6 +146,39 @@ namespace SabreTools.Core.Tools return path; } + /// + /// Get if the given path has a valid DAT extension + /// + /// Path to check + /// True if the extension is valid, false otherwise + public static bool HasValidDatExtension(string path) + { + // Get the extension from the path, if possible + string ext = Path.GetExtension(path).TrimStart('.'); + + // Check against the list of known DAT extensions + switch (ext) + { + case "csv": + case "dat": + case "json": + case "md5": + case "ripemd160": + case "sfv": + case "sha1": + case "sha256": + case "sha384": + case "sha512": + case "ssv": + case "tsv": + case "txt": + case "xml": + return true; + default: + return false; + } + } + /// Indicates whether the specified array is null or has a length of zero /// /// The array to test diff --git a/SabreTools.DatFiles/DatFile.cs b/SabreTools.DatFiles/DatFile.cs index 483ce5a6..c4d64c0a 100644 --- a/SabreTools.DatFiles/DatFile.cs +++ b/SabreTools.DatFiles/DatFile.cs @@ -16,17 +16,8 @@ namespace SabreTools.DatFiles /// /// Represents a format-agnostic DAT /// - /// - /// The fact that this one class could be separated into as many partial - /// classes as it did means that the functionality here should probably - /// be split out into either separate classes or even an entirely separate - /// namespace. Also, with that in mind, each of the individual DatFile types - /// probably should only need to inherit from a thin abstract class and - /// should not be exposed as part of the library, instead being taken care - /// of behind the scenes as part of the reading and writing. - /// [JsonObject("datfile"), XmlRoot("datfile")] - public abstract partial class DatFile + public abstract class DatFile { #region Fields diff --git a/SabreTools.DatFiles/DatHeader.cs b/SabreTools.DatFiles/DatHeader.cs index 8a28e71f..afc085d2 100644 --- a/SabreTools.DatFiles/DatHeader.cs +++ b/SabreTools.DatFiles/DatHeader.cs @@ -7,7 +7,6 @@ using System.Xml.Serialization; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatFiles.Formats; -using SabreTools.IO; using Newtonsoft.Json; namespace SabreTools.DatFiles @@ -1061,7 +1060,7 @@ namespace SabreTools.DatFiles string filename = string.IsNullOrWhiteSpace(FileName) ? Description : FileName; // Strip off the extension if it's a holdover from the DAT - if (Parser.HasValidDatExtension(filename)) + if (Utilities.HasValidDatExtension(filename)) filename = Path.GetFileNameWithoutExtension(filename); string outfile = $"{outDir}{filename}{extension}"; diff --git a/SabreTools.DatFiles/ItemDictionary.cs b/SabreTools.DatFiles/ItemDictionary.cs index 43c03486..2782d4d5 100644 --- a/SabreTools.DatFiles/ItemDictionary.cs +++ b/SabreTools.DatFiles/ItemDictionary.cs @@ -1,18 +1,13 @@ -using System; -using System.Collections; +using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Net; using System.Threading.Tasks; using System.Xml.Serialization; using SabreTools.Core; using SabreTools.DatItems; -using SabreTools.IO; using SabreTools.Logging; -using SabreTools.DatFiles.Reports; using NaturalSort; using Newtonsoft.Json; @@ -282,7 +277,7 @@ namespace SabreTools.DatFiles /// /// Special count only used by statistics output [JsonIgnore, XmlIgnore] - public long GameCount { get; private set; } = 0; + public long GameCount { get; set; } = 0; /// /// Total uncompressed size @@ -374,8 +369,6 @@ namespace SabreTools.DatFiles #endregion - #region Instance methods - #region Accessors /// @@ -462,6 +455,47 @@ namespace SabreTools.DatFiles } } + /// + /// Add statistics from another DatStats object + /// + /// DatStats object to add from + public void AddStatistics(ItemDictionary stats) + { + TotalCount += stats.Count; + + ArchiveCount += stats.ArchiveCount; + BiosSetCount += stats.BiosSetCount; + ChipCount += stats.ChipCount; + DiskCount += stats.DiskCount; + MediaCount += stats.MediaCount; + ReleaseCount += stats.ReleaseCount; + RomCount += stats.RomCount; + SampleCount += stats.SampleCount; + + GameCount += stats.GameCount; + + TotalSize += stats.TotalSize; + + // Individual hash counts + CRCCount += stats.CRCCount; + MD5Count += stats.MD5Count; +#if NET_FRAMEWORK + RIPEMD160Count += stats.RIPEMD160Count; +#endif + SHA1Count += stats.SHA1Count; + SHA256Count += stats.SHA256Count; + SHA384Count += stats.SHA384Count; + SHA512Count += stats.SHA512Count; + SpamSumCount += stats.SpamSumCount; + + // Individual status counts + BaddumpCount += stats.BaddumpCount; + GoodCount += stats.GoodCount; + NodumpCount += stats.NodumpCount; + RemovedCount += stats.RemovedCount; + VerifiedCount += stats.VerifiedCount; + } + /// /// Get if the file dictionary contains the key /// @@ -746,47 +780,6 @@ namespace SabreTools.DatFiles } } - /// - /// Add statistics from another DatStats object - /// - /// DatStats object to add from - private void AddStatistics(ItemDictionary stats) - { - TotalCount += stats.Count; - - ArchiveCount += stats.ArchiveCount; - BiosSetCount += stats.BiosSetCount; - ChipCount += stats.ChipCount; - DiskCount += stats.DiskCount; - MediaCount += stats.MediaCount; - ReleaseCount += stats.ReleaseCount; - RomCount += stats.RomCount; - SampleCount += stats.SampleCount; - - GameCount += stats.GameCount; - - TotalSize += stats.TotalSize; - - // Individual hash counts - CRCCount += stats.CRCCount; - MD5Count += stats.MD5Count; -#if NET_FRAMEWORK - RIPEMD160Count += stats.RIPEMD160Count; -#endif - SHA1Count += stats.SHA1Count; - SHA256Count += stats.SHA256Count; - SHA384Count += stats.SHA384Count; - SHA512Count += stats.SHA512Count; - SpamSumCount += stats.SpamSumCount; - - // Individual status counts - BaddumpCount += stats.BaddumpCount; - GoodCount += stats.GoodCount; - NodumpCount += stats.NodumpCount; - RemovedCount += stats.RemovedCount; - VerifiedCount += stats.VerifiedCount; - } - /// /// Ensure the key exists in the items dictionary /// @@ -1194,6 +1187,44 @@ namespace SabreTools.DatFiles } } + /// + /// Reset all statistics + /// + public void ResetStatistics() + { + TotalCount = 0; + + ArchiveCount = 0; + BiosSetCount = 0; + ChipCount = 0; + DiskCount = 0; + MediaCount = 0; + ReleaseCount = 0; + RomCount = 0; + SampleCount = 0; + + GameCount = 0; + + TotalSize = 0; + + CRCCount = 0; + MD5Count = 0; +#if NET_FRAMEWORK + RIPEMD160Count = 0; +#endif + SHA1Count = 0; + SHA256Count = 0; + SHA384Count = 0; + SHA512Count = 0; + SpamSumCount = 0; + + BaddumpCount = 0; + GoodCount = 0; + NodumpCount = 0; + RemovedCount = 0; + VerifiedCount = 0; + } + /// /// Get the highest-order Field value that represents the statistics /// @@ -1230,44 +1261,6 @@ namespace SabreTools.DatFiles return Field.DatItem_CRC; } - /// - /// Reset all statistics - /// - private void ResetStatistics() - { - TotalCount = 0; - - ArchiveCount = 0; - BiosSetCount = 0; - ChipCount = 0; - DiskCount = 0; - MediaCount = 0; - ReleaseCount = 0; - RomCount = 0; - SampleCount = 0; - - GameCount = 0; - - TotalSize = 0; - - CRCCount = 0; - MD5Count = 0; -#if NET_FRAMEWORK - RIPEMD160Count = 0; -#endif - SHA1Count = 0; - SHA256Count = 0; - SHA384Count = 0; - SHA512Count = 0; - SpamSumCount = 0; - - BaddumpCount = 0; - GoodCount = 0; - NodumpCount = 0; - RemovedCount = 0; - VerifiedCount = 0; - } - /// /// Sort the input DAT and get the key to be used by the item /// @@ -1335,216 +1328,5 @@ namespace SabreTools.DatFiles } #endregion - - #endregion // Instance methods - - #region Static methods - - #region Writing - - /// - /// Output the stats for a list of input dats as files in a human-readable format - /// - /// List of input files and folders - /// Name of the output file - /// True if single DAT stats are output, false otherwise - /// True if baddumps should be included in output, false otherwise - /// True if nodumps should be included in output, false otherwise - /// Set the statistics output format to use - public static void OutputStats( - List inputs, - string reportName, - string outDir, - bool single, - bool baddumpCol, - bool nodumpCol, - StatReportFormat statDatFormat) - { - // If there's no output format, set the default - if (statDatFormat == StatReportFormat.None) - statDatFormat = StatReportFormat.Textfile; - - // Get the proper output file name - if (string.IsNullOrWhiteSpace(reportName)) - reportName = "report"; - - // Get the proper output directory name - outDir = outDir.Ensure(); - - // Get the dictionary of desired output report names - Dictionary outputs = CreateOutStatsNames(outDir, statDatFormat, reportName); - - // Make sure we have all files and then order them - List files = PathTool.GetFilesOnly(inputs); - files = files - .OrderBy(i => Path.GetDirectoryName(i.CurrentPath)) - .ThenBy(i => Path.GetFileName(i.CurrentPath)) - .ToList(); - - // Get all of the writers that we need - List reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList(); - - // Write the header, if any - reports.ForEach(report => report.WriteHeader()); - - // Init all total variables - ItemDictionary totalStats = new ItemDictionary(); - - // Init directory-level variables - string lastdir = null; - string basepath = null; - ItemDictionary dirStats = new ItemDictionary(); - - // Now process each of the input files - foreach (ParentablePath file in files) - { - // Get the directory for the current file - string thisdir = Path.GetDirectoryName(file.CurrentPath); - basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath)); - - // If we don't have the first file and the directory has changed, show the previous directory stats and reset - if (lastdir != null && thisdir != lastdir) - { - // Output separator if needed - reports.ForEach(report => report.WriteMidSeparator()); - - DatFile lastdirdat = DatFile.Create(); - - reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); - reports.ForEach(report => report.Write()); - - // Write the mid-footer, if any - reports.ForEach(report => report.WriteFooterSeparator()); - - // Write the header, if any - reports.ForEach(report => report.WriteMidHeader()); - - // Reset the directory stats - dirStats.ResetStatistics(); - } - - staticLogger.Verbose($"Beginning stat collection for '{file.CurrentPath}'"); - List games = new List(); - DatFile datdata = Parser.CreateAndParse(file.CurrentPath); - datdata.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true); - - // Output single DAT stats (if asked) - staticLogger.User($"Adding stats for file '{file.CurrentPath}'\n"); - if (single) - { - reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items)); - reports.ForEach(report => report.Write()); - } - - // Add single DAT stats to dir - dirStats.AddStatistics(datdata.Items); - dirStats.GameCount += datdata.Items.Keys.Count(); - - // Add single DAT stats to totals - totalStats.AddStatistics(datdata.Items); - totalStats.GameCount += datdata.Items.Keys.Count(); - - // Make sure to assign the new directory - lastdir = thisdir; - } - - // Output the directory stats one last time - reports.ForEach(report => report.WriteMidSeparator()); - - if (single) - { - reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); - reports.ForEach(report => report.Write()); - } - - // Write the mid-footer, if any - reports.ForEach(report => report.WriteFooterSeparator()); - - // Write the header, if any - reports.ForEach(report => report.WriteMidHeader()); - - // Reset the directory stats - dirStats.ResetStatistics(); - - // Output total DAT stats - reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats)); - reports.ForEach(report => report.Write()); - - // Output footer if needed - reports.ForEach(report => report.WriteFooter()); - - staticLogger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen"); - } - - /// - /// Get the proper extension for the stat output format - /// - /// Output path to use - /// StatDatFormat to get the extension for - /// Name of the input file to use - /// Dictionary of output formats mapped to file names - private static Dictionary CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true) - { - Dictionary output = new Dictionary(); - - // First try to create the output directory if we need to - if (!Directory.Exists(outDir)) - Directory.CreateDirectory(outDir); - - // Double check the outDir for the end delim - if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString())) - outDir += Path.DirectorySeparatorChar; - - // For each output format, get the appropriate stream writer - output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite)); - - if (statDatFormat.HasFlag(StatReportFormat.Textfile)) - output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite)); - - if (statDatFormat.HasFlag(StatReportFormat.CSV)) - output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite)); - - if (statDatFormat.HasFlag(StatReportFormat.HTML)) - output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite)); - - if (statDatFormat.HasFlag(StatReportFormat.SSV)) - output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite)); - - if (statDatFormat.HasFlag(StatReportFormat.TSV)) - output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite)); - - return output; - } - - /// - /// Help generating the outstats name - /// - /// Output directory - /// Extension to use for the file - /// Name of the input file to use - /// True if we ignore existing files, false otherwise - /// String containing the new filename - private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite) - { - string outfile = outDir + reportName + extension; - outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); - - if (!overwrite) - { - int i = 1; - while (File.Exists(outfile)) - { - outfile = $"{outDir}{reportName}_{i}{extension}"; - outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); - i++; - } - } - - return outfile; - } - - #endregion - - #endregion // Static methods } } diff --git a/SabreTools.DatFiles/Reports/BaseReport.cs b/SabreTools.DatFiles/Reports/BaseReport.cs index f45dd6de..a1b32883 100644 --- a/SabreTools.DatFiles/Reports/BaseReport.cs +++ b/SabreTools.DatFiles/Reports/BaseReport.cs @@ -2,8 +2,8 @@ using System.IO; using SabreTools.Core; -using SabreTools.DatFiles; +// TODO: Reports namespace no longer is circular with DatFiles namespace SabreTools.DatFiles.Reports { /// diff --git a/SabreTools.DatFiles/SabreTools.DatFiles.csproj b/SabreTools.DatFiles/SabreTools.DatFiles.csproj index 04548183..5f12ac44 100644 --- a/SabreTools.DatFiles/SabreTools.DatFiles.csproj +++ b/SabreTools.DatFiles/SabreTools.DatFiles.csproj @@ -14,16 +14,12 @@ - - - - diff --git a/SabreTools.DatFiles/DatFromDir.cs b/SabreTools.DatTools/DatFromDir.cs similarity index 99% rename from SabreTools.DatFiles/DatFromDir.cs rename to SabreTools.DatTools/DatFromDir.cs index 1f462bd6..05f1fac8 100644 --- a/SabreTools.DatFiles/DatFromDir.cs +++ b/SabreTools.DatTools/DatFromDir.cs @@ -5,13 +5,14 @@ using System.Threading; using System.Threading.Tasks; using SabreTools.Core; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.FileTypes; using SabreTools.FileTypes.Archives; using SabreTools.IO; using SabreTools.Logging; -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { /// /// This file represents all methods related to populating a DatFile diff --git a/SabreTools.DatFiles/DatTool.cs b/SabreTools.DatTools/DatTool.cs similarity index 99% rename from SabreTools.DatFiles/DatTool.cs rename to SabreTools.DatTools/DatTool.cs index e4d0ac30..3330f1d2 100644 --- a/SabreTools.DatFiles/DatTool.cs +++ b/SabreTools.DatTools/DatTool.cs @@ -4,12 +4,12 @@ using System.Linq; using System.Threading.Tasks; using SabreTools.Core; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.IO; using SabreTools.Logging; -// TODO: Should each of the individual pieces of partial classes be their own classes? -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { // This file represents all methods related to converting and updating DatFiles public class DatTool diff --git a/SabreTools.DatFiles/Modification.cs b/SabreTools.DatTools/Modification.cs similarity index 99% rename from SabreTools.DatFiles/Modification.cs rename to SabreTools.DatTools/Modification.cs index 06e07dc9..5c3b8138 100644 --- a/SabreTools.DatFiles/Modification.cs +++ b/SabreTools.DatTools/Modification.cs @@ -7,13 +7,14 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using SabreTools.Core; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.Filtering; using SabreTools.IO; using SabreTools.Logging; // This file represents all methods related to the Filtering namespace -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { public class Modification { diff --git a/SabreTools.DatFiles/Parser.cs b/SabreTools.DatTools/Parser.cs similarity index 89% rename from SabreTools.DatFiles/Parser.cs rename to SabreTools.DatTools/Parser.cs index d774b733..43cd0a99 100644 --- a/SabreTools.DatFiles/Parser.cs +++ b/SabreTools.DatTools/Parser.cs @@ -3,11 +3,13 @@ using System.IO; using System.Text.RegularExpressions; using SabreTools.Core; +using SabreTools.Core.Tools; +using SabreTools.DatFiles; using SabreTools.IO; using SabreTools.Logging; // This file represents all methods related to parsing from a file -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { public class Parser { @@ -78,7 +80,7 @@ namespace SabreTools.DatFiles string currentPath = input.CurrentPath; // Check the file extension first as a safeguard - if (!HasValidDatExtension(currentPath)) + if (!Utilities.HasValidDatExtension(currentPath)) return; // If the output filename isn't set already, get the internal filename @@ -105,39 +107,6 @@ namespace SabreTools.DatFiles } } - /// - /// Get if the given path has a valid DAT extension - /// - /// Path to check - /// True if the extension is valid, false otherwise - public static bool HasValidDatExtension(string path) - { - // Get the extension from the path, if possible - string ext = path.GetNormalizedExtension(); - - // Check against the list of known DAT extensions - switch (ext) - { - case "csv": - case "dat": - case "json": - case "md5": - case "ripemd160": - case "sfv": - case "sha1": - case "sha256": - case "sha384": - case "sha512": - case "ssv": - case "tsv": - case "txt": - case "xml": - return true; - default: - return false; - } - } - /// /// Get what type of DAT the input file is /// @@ -146,7 +115,7 @@ namespace SabreTools.DatFiles private static DatFormat GetDatFormat(string filename) { // Limit the output formats based on extension - if (!HasValidDatExtension(filename)) + if (!Utilities.HasValidDatExtension(filename)) return 0; // Get the extension from the filename diff --git a/SabreTools.DatFiles/Rebuilder.cs b/SabreTools.DatTools/Rebuilder.cs similarity index 99% rename from SabreTools.DatFiles/Rebuilder.cs rename to SabreTools.DatTools/Rebuilder.cs index 68991e9b..864d4327 100644 --- a/SabreTools.DatFiles/Rebuilder.cs +++ b/SabreTools.DatTools/Rebuilder.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using SabreTools.Core; using SabreTools.Core.Tools; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.FileTypes; using SabreTools.FileTypes.Archives; @@ -13,7 +14,7 @@ using SabreTools.Logging; using SabreTools.Skippers; // This file represents all methods related to rebuilding from a DatFile -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { public class Rebuilder { diff --git a/SabreTools.DatTools/SabreTools.DatTools.csproj b/SabreTools.DatTools/SabreTools.DatTools.csproj new file mode 100644 index 00000000..0d84581b --- /dev/null +++ b/SabreTools.DatTools/SabreTools.DatTools.csproj @@ -0,0 +1,25 @@ + + + + net48;netcoreapp3.1;net5.0 + win10-x64;win7-x86 + Debug;Release + AnyCPU;x64 + + + + NET_FRAMEWORK + + + + + + + + + + + + + + diff --git a/SabreTools.DatFiles/Splitter.cs b/SabreTools.DatTools/Splitter.cs similarity index 99% rename from SabreTools.DatFiles/Splitter.cs rename to SabreTools.DatTools/Splitter.cs index 8da17071..7df0ad30 100644 --- a/SabreTools.DatFiles/Splitter.cs +++ b/SabreTools.DatTools/Splitter.cs @@ -6,13 +6,14 @@ using System.Net; using System.Threading.Tasks; using SabreTools.Core; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.IO; using SabreTools.Logging; using NaturalSort; // This file represents all methods related to splitting a DatFile into multiple -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { // TODO: Implement Level split public class Splitter diff --git a/SabreTools.DatTools/Statistics.cs b/SabreTools.DatTools/Statistics.cs new file mode 100644 index 00000000..997ef62b --- /dev/null +++ b/SabreTools.DatTools/Statistics.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; + +using SabreTools.Core; +using SabreTools.DatFiles; +using SabreTools.DatFiles.Reports; +using SabreTools.IO; +using SabreTools.Logging; + +namespace SabreTools.DatTools +{ + public class Statistics + { + #region Logging + + /// + /// Logging object + /// + private static readonly Logger logger = new Logger(); + + #endregion + + /// + /// Output the stats for a list of input dats as files in a human-readable format + /// + /// List of input files and folders + /// Name of the output file + /// True if single DAT stats are output, false otherwise + /// True if baddumps should be included in output, false otherwise + /// True if nodumps should be included in output, false otherwise + /// Set the statistics output format to use + public static void OutputStats( + List inputs, + string reportName, + string outDir, + bool single, + bool baddumpCol, + bool nodumpCol, + StatReportFormat statDatFormat) + { + // If there's no output format, set the default + if (statDatFormat == StatReportFormat.None) + statDatFormat = StatReportFormat.Textfile; + + // Get the proper output file name + if (string.IsNullOrWhiteSpace(reportName)) + reportName = "report"; + + // Get the proper output directory name + outDir = outDir.Ensure(); + + // Get the dictionary of desired output report names + Dictionary outputs = CreateOutStatsNames(outDir, statDatFormat, reportName); + + // Make sure we have all files and then order them + List files = PathTool.GetFilesOnly(inputs); + files = files + .OrderBy(i => Path.GetDirectoryName(i.CurrentPath)) + .ThenBy(i => Path.GetFileName(i.CurrentPath)) + .ToList(); + + // Get all of the writers that we need + List reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList(); + + // Write the header, if any + reports.ForEach(report => report.WriteHeader()); + + // Init all total variables + ItemDictionary totalStats = new ItemDictionary(); + + // Init directory-level variables + string lastdir = null; + string basepath = null; + ItemDictionary dirStats = new ItemDictionary(); + + // Now process each of the input files + foreach (ParentablePath file in files) + { + // Get the directory for the current file + string thisdir = Path.GetDirectoryName(file.CurrentPath); + basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath)); + + // If we don't have the first file and the directory has changed, show the previous directory stats and reset + if (lastdir != null && thisdir != lastdir) + { + // Output separator if needed + reports.ForEach(report => report.WriteMidSeparator()); + + DatFile lastdirdat = DatFile.Create(); + + reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); + reports.ForEach(report => report.Write()); + + // Write the mid-footer, if any + reports.ForEach(report => report.WriteFooterSeparator()); + + // Write the header, if any + reports.ForEach(report => report.WriteMidHeader()); + + // Reset the directory stats + dirStats.ResetStatistics(); + } + + logger.Verbose($"Beginning stat collection for '{file.CurrentPath}'"); + List games = new List(); + DatFile datdata = Parser.CreateAndParse(file.CurrentPath); + datdata.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true); + + // Output single DAT stats (if asked) + logger.User($"Adding stats for file '{file.CurrentPath}'\n"); + if (single) + { + reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items)); + reports.ForEach(report => report.Write()); + } + + // Add single DAT stats to dir + dirStats.AddStatistics(datdata.Items); + dirStats.GameCount += datdata.Items.Keys.Count(); + + // Add single DAT stats to totals + totalStats.AddStatistics(datdata.Items); + totalStats.GameCount += datdata.Items.Keys.Count(); + + // Make sure to assign the new directory + lastdir = thisdir; + } + + // Output the directory stats one last time + reports.ForEach(report => report.WriteMidSeparator()); + + if (single) + { + reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); + reports.ForEach(report => report.Write()); + } + + // Write the mid-footer, if any + reports.ForEach(report => report.WriteFooterSeparator()); + + // Write the header, if any + reports.ForEach(report => report.WriteMidHeader()); + + // Reset the directory stats + dirStats.ResetStatistics(); + + // Output total DAT stats + reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats)); + reports.ForEach(report => report.Write()); + + // Output footer if needed + reports.ForEach(report => report.WriteFooter()); + + logger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen"); + } + + /// + /// Get the proper extension for the stat output format + /// + /// Output path to use + /// StatDatFormat to get the extension for + /// Name of the input file to use + /// Dictionary of output formats mapped to file names + private static Dictionary CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true) + { + Dictionary output = new Dictionary(); + + // First try to create the output directory if we need to + if (!Directory.Exists(outDir)) + Directory.CreateDirectory(outDir); + + // Double check the outDir for the end delim + if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString())) + outDir += Path.DirectorySeparatorChar; + + // For each output format, get the appropriate stream writer + output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite)); + + if (statDatFormat.HasFlag(StatReportFormat.Textfile)) + output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite)); + + if (statDatFormat.HasFlag(StatReportFormat.CSV)) + output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite)); + + if (statDatFormat.HasFlag(StatReportFormat.HTML)) + output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite)); + + if (statDatFormat.HasFlag(StatReportFormat.SSV)) + output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite)); + + if (statDatFormat.HasFlag(StatReportFormat.TSV)) + output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite)); + + return output; + } + + /// + /// Help generating the outstats name + /// + /// Output directory + /// Extension to use for the file + /// Name of the input file to use + /// True if we ignore existing files, false otherwise + /// String containing the new filename + private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite) + { + string outfile = outDir + reportName + extension; + outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); + + if (!overwrite) + { + int i = 1; + while (File.Exists(outfile)) + { + outfile = $"{outDir}{reportName}_{i}{extension}"; + outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); + i++; + } + } + + return outfile; + } + } +} diff --git a/SabreTools.DatFiles/Verification.cs b/SabreTools.DatTools/Verification.cs similarity index 99% rename from SabreTools.DatFiles/Verification.cs rename to SabreTools.DatTools/Verification.cs index ea9efaed..cb4a4140 100644 --- a/SabreTools.DatFiles/Verification.cs +++ b/SabreTools.DatTools/Verification.cs @@ -4,13 +4,14 @@ using System.Linq; using SabreTools.Core; using SabreTools.Core.Tools; +using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.FileTypes; using SabreTools.FileTypes.Archives; using SabreTools.Logging; // This file represents all methods related to verifying with a DatFile -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { public class Verification { diff --git a/SabreTools.DatFiles/Writer.cs b/SabreTools.DatTools/Writer.cs similarity index 99% rename from SabreTools.DatFiles/Writer.cs rename to SabreTools.DatTools/Writer.cs index 960c3f7c..b44943fb 100644 --- a/SabreTools.DatFiles/Writer.cs +++ b/SabreTools.DatTools/Writer.cs @@ -4,12 +4,13 @@ using System.Linq; using System.Threading.Tasks; using SabreTools.Core; +using SabreTools.DatFiles; using SabreTools.DatFiles.Reports; using SabreTools.IO; using SabreTools.Logging; // This file represents all methods related to writing to a file -namespace SabreTools.DatFiles +namespace SabreTools.DatTools { public class Writer { diff --git a/SabreTools.sln b/SabreTools.sln index e54f6b58..cabc4a6c 100644 --- a/SabreTools.sln +++ b/SabreTools.sln @@ -32,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.FileTypes", "Sab EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.DatItems", "SabreTools.DatItems\SabreTools.DatItems.csproj", "{90ADE461-33B1-4E0D-925F-C99913665F0C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.DatTools", "SabreTools.DatTools\SabreTools.DatTools.csproj", "{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -128,6 +130,14 @@ Global {90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|Any CPU.Build.0 = Release|Any CPU {90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|x64.ActiveCfg = Release|Any CPU {90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|x64.Build.0 = Release|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|x64.ActiveCfg = Debug|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|x64.Build.0 = Debug|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|Any CPU.Build.0 = Release|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|x64.ActiveCfg = Release|Any CPU + {E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/SabreTools/Features/Batch.cs b/SabreTools/Features/Batch.cs index 53604f41..747d1879 100644 --- a/SabreTools/Features/Batch.cs +++ b/SabreTools/Features/Batch.cs @@ -7,6 +7,7 @@ using System.Text.RegularExpressions; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Filtering; using SabreTools.Help; using SabreTools.IO; @@ -153,7 +154,7 @@ Reset the internal state: reset();"; // Assume there could be multiple foreach (string input in command.Arguments) { - DatFiles.DatFromDir.PopulateFromDir(datFile, input); + DatTools.DatFromDir.PopulateFromDir(datFile, input); } // TODO: We might not want to remove higher order hashes in the future diff --git a/SabreTools/Features/DatFromDir.cs b/SabreTools/Features/DatFromDir.cs index b5b8f344..d936eac0 100644 --- a/SabreTools/Features/DatFromDir.cs +++ b/SabreTools/Features/DatFromDir.cs @@ -4,7 +4,9 @@ using System.IO; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; + namespace SabreTools.Features { internal class DatFromDir : BaseFeature @@ -89,7 +91,7 @@ namespace SabreTools.Features datdata.FillHeaderFromPath(basePath, noAutomaticDate); // Now populate from the path - bool success = DatFiles.DatFromDir.PopulateFromDir( + bool success = DatTools.DatFromDir.PopulateFromDir( datdata, basePath, asFiles, diff --git a/SabreTools/Features/Sort.cs b/SabreTools/Features/Sort.cs index 34177984..9046d396 100644 --- a/SabreTools/Features/Sort.cs +++ b/SabreTools/Features/Sort.cs @@ -3,6 +3,7 @@ using System.IO; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; using SabreTools.Logging; diff --git a/SabreTools/Features/Split.cs b/SabreTools/Features/Split.cs index d6fa1675..d7a83a05 100644 --- a/SabreTools/Features/Split.cs +++ b/SabreTools/Features/Split.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; using SabreTools.Logging; diff --git a/SabreTools/Features/Stats.cs b/SabreTools/Features/Stats.cs index 3c73d582..54c36423 100644 --- a/SabreTools/Features/Stats.cs +++ b/SabreTools/Features/Stats.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.IO; -using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; namespace SabreTools.Features @@ -55,7 +55,7 @@ The stats that are outputted are as follows: filename = Path.GetFileName(filename); } - ItemDictionary.OutputStats( + Statistics.OutputStats( Inputs, filename, OutputDir, diff --git a/SabreTools/Features/Update.cs b/SabreTools/Features/Update.cs index b8a1a44f..124cc36d 100644 --- a/SabreTools/Features/Update.cs +++ b/SabreTools/Features/Update.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; using SabreTools.Logging; diff --git a/SabreTools/Features/Verify.cs b/SabreTools/Features/Verify.cs index 8a4c29ba..0f680ced 100644 --- a/SabreTools/Features/Verify.cs +++ b/SabreTools/Features/Verify.cs @@ -2,6 +2,7 @@ using SabreTools.Core; using SabreTools.DatFiles; +using SabreTools.DatTools; using SabreTools.Help; using SabreTools.IO; using SabreTools.Logging; @@ -84,7 +85,7 @@ namespace SabreTools.Features logger.User("Processing files:\n"); foreach (string input in Inputs) { - DatFiles.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); + DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); } Verification.VerifyGeneric(datdata, hashOnly); @@ -133,7 +134,7 @@ namespace SabreTools.Features logger.User("Processing files:\n"); foreach (string input in Inputs) { - DatFiles.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); + DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); } Verification.VerifyGeneric(datdata, hashOnly); diff --git a/SabreTools/SabreTools.csproj b/SabreTools/SabreTools.csproj index 8991fabb..1eb1e709 100644 --- a/SabreTools/SabreTools.csproj +++ b/SabreTools/SabreTools.csproj @@ -18,6 +18,7 @@ +