using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using SabreTools.Library.Data; using SabreTools.Library.DatItems; using SabreTools.Library.Reports; using SabreTools.Library.Tools; namespace SabreTools.Library.DatFiles { /// /// Represents statistical data associated with a DAT /// public class DatStats { #region Private instance variables /// /// Object used to lock stats updates /// private readonly object _lockObject = new object(); #endregion #region Publicly facing variables /// /// Statistics writing format /// public StatReportFormat ReportFormat { get; set; } = StatReportFormat.None; /// /// Statistics output file name /// public string ReportName { get; set; } = "report"; /// /// Overall item count /// public long Count { get; set; } = 0; /// /// Number of Archive items /// public long ArchiveCount { get; set; } = 0; /// /// Number of BiosSet items /// public long BiosSetCount { get; set; } = 0; /// /// Number of Disk items /// public long DiskCount { get; set; } = 0; /// /// Number of Release items /// public long ReleaseCount { get; set; } = 0; /// /// Number of Rom items /// public long RomCount { get; set; } = 0; /// /// Number of Sample items /// public long SampleCount { get; set; } = 0; /// /// Number of machines /// /// Special count only used by statistics output public long GameCount { get; set; } = 0; /// /// Total uncompressed size /// public long TotalSize { get; set; } = 0; /// /// Number of items with a CRC hash /// public long CRCCount { get; set; } = 0; /// /// Number of items with an MD5 hash /// public long MD5Count { get; set; } = 0; #if NET_FRAMEWORK /// /// Number of items with a RIPEMD160 hash /// public long RIPEMD160Count { get; set; } = 0; #endif /// /// Number of items with a SHA-1 hash /// public long SHA1Count { get; set; } = 0; /// /// Number of items with a SHA-256 hash /// public long SHA256Count { get; set; } = 0; /// /// Number of items with a SHA-384 hash /// public long SHA384Count { get; set; } = 0; /// /// Number of items with a SHA-512 hash /// public long SHA512Count { get; set; } = 0; /// /// Number of items with the baddump status /// public long BaddumpCount { get; set; } = 0; /// /// Number of items with the good status /// public long GoodCount { get; set; } = 0; /// /// Number of items with the nodump status /// public long NodumpCount { get; set; } = 0; /// /// Number of items with the verified status /// public long VerifiedCount { get; set; } = 0; #endregion #region Instance Methods #region Updates /// /// Add to the statistics given a DatItem /// /// Item to add info from public void AddItem(DatItem item) { // No matter what the item is, we increate the count lock (_lockObject) { this.Count += 1; // Now we do different things for each item type switch (item.ItemType) { case ItemType.Archive: this.ArchiveCount += 1; break; case ItemType.BiosSet: this.BiosSetCount += 1; break; case ItemType.Disk: this.DiskCount += 1; if (((Disk)item).ItemStatus != ItemStatus.Nodump) { this.MD5Count += (string.IsNullOrWhiteSpace(((Disk)item).MD5) ? 0 : 1); #if NET_FRAMEWORK this.RIPEMD160Count += (string.IsNullOrWhiteSpace(((Disk)item).RIPEMD160) ? 0 : 1); #endif this.SHA1Count += (string.IsNullOrWhiteSpace(((Disk)item).SHA1) ? 0 : 1); this.SHA256Count += (string.IsNullOrWhiteSpace(((Disk)item).SHA256) ? 0 : 1); this.SHA384Count += (string.IsNullOrWhiteSpace(((Disk)item).SHA384) ? 0 : 1); this.SHA512Count += (string.IsNullOrWhiteSpace(((Disk)item).SHA512) ? 0 : 1); } this.BaddumpCount += (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); this.GoodCount += (((Disk)item).ItemStatus == ItemStatus.Good ? 1 : 0); this.NodumpCount += (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); this.VerifiedCount += (((Disk)item).ItemStatus == ItemStatus.Verified ? 1 : 0); break; case ItemType.Release: this.ReleaseCount += 1; break; case ItemType.Rom: this.RomCount += 1; if (((Rom)item).ItemStatus != ItemStatus.Nodump) { this.TotalSize += ((Rom)item).Size; this.CRCCount += (string.IsNullOrWhiteSpace(((Rom)item).CRC) ? 0 : 1); this.MD5Count += (string.IsNullOrWhiteSpace(((Rom)item).MD5) ? 0 : 1); #if NET_FRAMEWORK this.RIPEMD160Count += (string.IsNullOrWhiteSpace(((Rom)item).RIPEMD160) ? 0 : 1); #endif this.SHA1Count += (string.IsNullOrWhiteSpace(((Rom)item).SHA1) ? 0 : 1); this.SHA256Count += (string.IsNullOrWhiteSpace(((Rom)item).SHA256) ? 0 : 1); this.SHA384Count += (string.IsNullOrWhiteSpace(((Rom)item).SHA384) ? 0 : 1); this.SHA512Count += (string.IsNullOrWhiteSpace(((Rom)item).SHA512) ? 0 : 1); } this.BaddumpCount += (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); this.GoodCount += (((Rom)item).ItemStatus == ItemStatus.Good ? 1 : 0); this.NodumpCount += (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); this.VerifiedCount += (((Rom)item).ItemStatus == ItemStatus.Verified ? 1 : 0); break; case ItemType.Sample: this.SampleCount += 1; break; } } } /// /// Add statistics from another DatStats object /// /// DatStats object to add from public void AddStats(DatStats stats) { this.Count += stats.Count; this.ArchiveCount += stats.ArchiveCount; this.BiosSetCount += stats.BiosSetCount; this.DiskCount += stats.DiskCount; this.ReleaseCount += stats.ReleaseCount; this.RomCount += stats.RomCount; this.SampleCount += stats.SampleCount; this.GameCount += stats.GameCount; this.TotalSize += stats.TotalSize; // Individual hash counts this.CRCCount += stats.CRCCount; this.MD5Count += stats.MD5Count; #if NET_FRAMEWORK this.RIPEMD160Count += stats.RIPEMD160Count; #endif this.SHA1Count += stats.SHA1Count; this.SHA256Count += stats.SHA256Count; this.SHA384Count += stats.SHA384Count; this.SHA512Count += stats.SHA512Count; // Individual status counts this.BaddumpCount += stats.BaddumpCount; this.GoodCount += stats.GoodCount; this.NodumpCount += stats.NodumpCount; this.VerifiedCount += stats.VerifiedCount; } /// /// Get the highest-order BucketedBy value that represents the statistics /// public BucketedBy GetBestAvailable() { // If all items are supposed to have a SHA-512, we bucket by that if (RomCount + DiskCount - NodumpCount == SHA512Count) return BucketedBy.SHA512; // If all items are supposed to have a SHA-384, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA384Count) return BucketedBy.SHA384; // If all items are supposed to have a SHA-256, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA256Count) return BucketedBy.SHA256; // If all items are supposed to have a SHA-1, we bucket by that else if (RomCount + DiskCount - NodumpCount == SHA1Count) return BucketedBy.SHA1; #if NET_FRAMEWORK // If all items are supposed to have a RIPEMD160, we bucket by that else if (RomCount + DiskCount - NodumpCount == RIPEMD160Count) return BucketedBy.RIPEMD160; #endif // If all items are supposed to have a MD5, we bucket by that else if (RomCount + DiskCount - NodumpCount == MD5Count) return BucketedBy.MD5; // Otherwise, we bucket by CRC else return BucketedBy.CRC; } /// /// Remove from the statistics given a DatItem /// /// Item to remove info for public void RemoveItem(DatItem item) { // No matter what the item is, we increate the count lock (_lockObject) { this.Count -= 1; // Now we do different things for each item type switch (item.ItemType) { case ItemType.Archive: this.ArchiveCount -= 1; break; case ItemType.BiosSet: this.BiosSetCount -= 1; break; case ItemType.Disk: this.DiskCount -= 1; if (((Disk)item).ItemStatus != ItemStatus.Nodump) { this.MD5Count -= (string.IsNullOrWhiteSpace(((Disk)item).MD5) ? 0 : 1); #if NET_FRAMEWORK this.RIPEMD160Count -= (string.IsNullOrWhiteSpace(((Disk)item).RIPEMD160) ? 0 : 1); #endif this.SHA1Count -= (string.IsNullOrWhiteSpace(((Disk)item).SHA1) ? 0 : 1); this.SHA256Count -= (string.IsNullOrWhiteSpace(((Disk)item).SHA256) ? 0 : 1); this.SHA384Count -= (string.IsNullOrWhiteSpace(((Disk)item).SHA384) ? 0 : 1); this.SHA512Count -= (string.IsNullOrWhiteSpace(((Disk)item).SHA512) ? 0 : 1); } this.BaddumpCount -= (((Disk)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); this.GoodCount -= (((Disk)item).ItemStatus == ItemStatus.Good ? 1 : 0); this.NodumpCount -= (((Disk)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); this.VerifiedCount -= (((Disk)item).ItemStatus == ItemStatus.Verified ? 1 : 0); break; case ItemType.Release: this.ReleaseCount -= 1; break; case ItemType.Rom: this.RomCount -= 1; if (((Rom)item).ItemStatus != ItemStatus.Nodump) { this.TotalSize -= ((Rom)item).Size; this.CRCCount -= (string.IsNullOrWhiteSpace(((Rom)item).CRC) ? 0 : 1); this.MD5Count -= (string.IsNullOrWhiteSpace(((Rom)item).MD5) ? 0 : 1); #if NET_FRAMEWORK this.RIPEMD160Count -= (string.IsNullOrWhiteSpace(((Rom)item).RIPEMD160) ? 0 : 1); #endif this.SHA1Count -= (string.IsNullOrWhiteSpace(((Rom)item).SHA1) ? 0 : 1); this.SHA256Count -= (string.IsNullOrWhiteSpace(((Rom)item).SHA256) ? 0 : 1); this.SHA384Count -= (string.IsNullOrWhiteSpace(((Rom)item).SHA384) ? 0 : 1); this.SHA512Count -= (string.IsNullOrWhiteSpace(((Rom)item).SHA512) ? 0 : 1); } this.BaddumpCount -= (((Rom)item).ItemStatus == ItemStatus.BadDump ? 1 : 0); this.GoodCount -= (((Rom)item).ItemStatus == ItemStatus.Good ? 1 : 0); this.NodumpCount -= (((Rom)item).ItemStatus == ItemStatus.Nodump ? 1 : 0); this.VerifiedCount -= (((Rom)item).ItemStatus == ItemStatus.Verified ? 1 : 0); break; case ItemType.Sample: this.SampleCount -= 1; break; } } } /// /// Reset all statistics /// public void Reset() { this.Count = 0; this.ArchiveCount = 0; this.BiosSetCount = 0; this.DiskCount = 0; this.ReleaseCount = 0; this.RomCount = 0; this.SampleCount = 0; this.GameCount = 0; this.TotalSize = 0; this.CRCCount = 0; this.MD5Count = 0; #if NET_FRAMEWORK this.RIPEMD160Count = 0; #endif this.SHA1Count = 0; this.SHA256Count = 0; this.SHA384Count = 0; this.SHA512Count = 0; this.BaddumpCount = 0; this.GoodCount = 0; this.NodumpCount = 0; this.VerifiedCount = 0; } #endregion #region Writing /// /// Output the stats for a list of input dats as files in a human-readable format /// /// List of input files and folders /// Name of the output file /// True if single DAT stats are output, false otherwise /// True if baddumps should be included in output, false otherwise /// True if nodumps should be included in output, false otherwise /// Set the statistics output format to use public static void OutputStats(List inputs, string reportName, string outDir, bool single, bool baddumpCol, bool nodumpCol, StatReportFormat statDatFormat) { // If there's no output format, set the default if (statDatFormat == StatReportFormat.None) statDatFormat = StatReportFormat.Textfile; // Get the proper output file name if (string.IsNullOrWhiteSpace(reportName)) reportName = "report"; // Get the proper output directory name outDir = DirectoryExtensions.Ensure(outDir); // Get the dictionary of desired output report names Dictionary outputs = DatStats.CreateOutStatsNames(outDir, statDatFormat, reportName); // Make sure we have all files and then order them List files = DirectoryExtensions.GetFilesOnly(inputs); files = files .OrderBy(i => Path.GetDirectoryName(i)) .ThenBy(i => Path.GetFileName(i)) .ToList(); // Get all of the writers that we need List reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList(); // Write the header, if any reports.ForEach(report => report.WriteHeader()); // Init all total variables DatStats totalStats = new DatStats(); // Init directory-level variables string lastdir = null; string basepath = null; DatStats dirStats = new DatStats(); // Now process each of the input files foreach (string file in files) { // Get the directory for the current file string thisdir = Path.GetDirectoryName(file); basepath = Path.GetDirectoryName(Path.GetDirectoryName(file)); // If we don't have the first file and the directory has changed, show the previous directory stats and reset if (lastdir != null && thisdir != lastdir) { // Output separator if needed reports.ForEach(report => report.WriteMidSeparator()); DatFile lastdirdat = DatFile.Create(); reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); reports.ForEach(report => report.Write()); // Write the mid-footer, if any reports.ForEach(report => report.WriteFooterSeparator()); // Write the header, if any reports.ForEach(report => report.WriteMidHeader()); // Reset the directory stats dirStats.Reset(); } Globals.Logger.Verbose($"Beginning stat collection for '{file}'", false); List games = new List(); DatFile datdata = DatFile.CreateAndParse(file); datdata.BucketBy(BucketedBy.Game, DedupeType.None, norename: true); // Output single DAT stats (if asked) Globals.Logger.User($"Adding stats for file '{file}'\n", false); if (single) { reports.ForEach(report => report.ReplaceStatistics(datdata.DatHeader.FileName, datdata.Keys.Count, datdata.DatStats)); reports.ForEach(report => report.Write()); } // Add single DAT stats to dir dirStats.AddStats(datdata.DatStats); dirStats.GameCount += datdata.Keys.Count(); // Add single DAT stats to totals totalStats.AddStats(datdata.DatStats); totalStats.GameCount += datdata.Keys.Count(); // Make sure to assign the new directory lastdir = thisdir; } // Output the directory stats one last time reports.ForEach(report => report.WriteMidSeparator()); if (single) { reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); reports.ForEach(report => report.Write()); } // Write the mid-footer, if any reports.ForEach(report => report.WriteFooterSeparator()); // Write the header, if any reports.ForEach(report => report.WriteMidHeader()); // Reset the directory stats dirStats.Reset(); // Output total DAT stats reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats)); reports.ForEach(report => report.Write()); // Output footer if needed reports.ForEach(report => report.WriteFooter()); Globals.Logger.User(@" Please check the log folder if the stats scrolled offscreen", false); } /// /// Get the proper extension for the stat output format /// /// Output path to use /// StatDatFormat to get the extension for /// Name of the input file to use /// Dictionary of output formats mapped to file names private static Dictionary CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true) { Dictionary output = new Dictionary(); // First try to create the output directory if we need to if (!Directory.Exists(outDir)) Directory.CreateDirectory(outDir); // Double check the outDir for the end delim if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString())) outDir += Path.DirectorySeparatorChar; // For each output format, get the appropriate stream writer if (statDatFormat.HasFlag(StatReportFormat.None)) output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite)); if (statDatFormat.HasFlag(StatReportFormat.Textfile)) output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite)); if (statDatFormat.HasFlag(StatReportFormat.CSV)) output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite)); if (statDatFormat.HasFlag(StatReportFormat.HTML)) output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite)); if (statDatFormat.HasFlag(StatReportFormat.SSV)) output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite)); if (statDatFormat.HasFlag(StatReportFormat.TSV)) output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite)); return output; } /// /// Help generating the outstats name /// /// Output directory /// Extension to use for the file /// Name of the input file to use /// True if we ignore existing files, false otherwise /// String containing the new filename private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite) { string outfile = outDir + reportName + extension; outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); if (!overwrite) { int i = 1; while (File.Exists(outfile)) { outfile = $"{outDir}{reportName}_{i}{extension}"; outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString()); i++; } } return outfile; } #endregion #endregion // Instance Methods } }