diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index 09af9762..efaef197 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -143,9 +143,11 @@ namespace SabreTools.Helper helptext.Add(" -out= Output directory"); helptext.Add(" -hs, --hash-split Split a DAT or folder by best-available hashes"); helptext.Add(" -out= Output directory"); - helptext.Add(" -html, --html Write statistics on all input DATs to HTML"); helptext.Add(" -st, --stats Get statistics on all input DATs"); + helptext.Add(" -csv, --csv Output in Comma-Separated Value format"); + helptext.Add(" -html, --html Write stats to HTML"); helptext.Add(" -si, --single Show individual statistics"); + helptext.Add(" -tsv, --tsv Output in Tab-Separated Value format"); helptext.Add(" -ts, --type-split Split a DAT or folder by file types (rom/disk)"); helptext.Add(" -out= Output directory"); helptext.Add(" -ud, --update Update a DAT file"); diff --git a/SabreTools.Helper/Data/Enums.cs b/SabreTools.Helper/Data/Enums.cs index 9a4dfc40..dafe8081 100644 --- a/SabreTools.Helper/Data/Enums.cs +++ b/SabreTools.Helper/Data/Enums.cs @@ -1,26 +1,5 @@ namespace SabreTools.Helper { - #region DATabase - - /// - /// Possible DAT import classes - /// - public enum DatType - { - none = 0, - Custom, - MAME, - NoIntro, - Redump, - TOSEC, - TruRip, - NonGood, - MaybeIntro, - Good, - } - - #endregion - #region DatFile related /// @@ -97,6 +76,17 @@ NotNodump = 5, // This is a fake flag that is used for filter only } + /// + /// Determine which format to output Stats to + /// + public enum StatOutputFormat + { + None = 0, + HTML = 1, + CSV = 2, + TSV = 3, + } + #endregion #region Skippers and Mappers diff --git a/SabreTools.Helper/Objects/Dat/DatFile.cs b/SabreTools.Helper/Objects/Dat/DatFile.cs index 340206eb..0771f6c9 100644 --- a/SabreTools.Helper/Objects/Dat/DatFile.cs +++ b/SabreTools.Helper/Objects/Dat/DatFile.cs @@ -3003,7 +3003,9 @@ namespace SabreTools.Helper // Output initial statistics, for kicks if (stats) { - OutputStats(logger, logger, (RomCount + DiskCount == 0)); + StreamWriter sw = new StreamWriter(new MemoryStream()); + OutputStats(sw, StatOutputFormat.None, logger, (RomCount + DiskCount == 0)); + sw.Dispose(); } // Bucket roms by game name and optionally dedupe @@ -4335,11 +4337,12 @@ namespace SabreTools.Helper /// /// Output the stats for the Dat in a human-readable format /// + /// StreamWriter representing the output file or stream for the statistics + /// Set the statistics output format to use /// Logger object for file and console writing - /// Logger object for file and console output (statistics) /// True if numbers should be recalculated for the DAT, false otherwise (default) /// Number of games to use, -1 means recalculate games (default) - public void OutputStats(Logger logger, Logger statLogger, bool recalculate = false, long game = -1) + public void OutputStats(StreamWriter sw, StatOutputFormat statOutputFormat, Logger logger, bool recalculate = false, long game = -1) { // If we're supposed to recalculate the statistics, do so if (recalculate) @@ -4352,7 +4355,11 @@ namespace SabreTools.Helper { TotalSize = Int64.MaxValue + TotalSize; } - statLogger.User(" Uncompressed size: " + Style.GetBytesReadable(TotalSize) + @" + + // Log the results to screen + logger.User(@"For file '" + FileName + @"': +-------------------------------------------------- + Uncompressed size: " + Style.GetBytesReadable(TotalSize) + @" Games found: " + (game == -1 ? Files.Count : game) + @" Roms found: " + RomCount + @" Disks found: " + DiskCount + @" @@ -4360,39 +4367,66 @@ namespace SabreTools.Helper Roms with MD5: " + MD5Count + @" Roms with SHA-1: " + SHA1Count + @" Roms with Nodump status: " + NodumpCount + @" -", false); - } - /// - /// Output the stats for the Dat in a human-readable HTML format - /// - /// StreamWriter representing the output file or stream for the statistics - /// Logger object for file and console writing - /// True if numbers should be recalculated for the DAT, false otherwise (default) - /// Number of games to use, -1 means recalculate games (default) - public void OutputHTMLStats(StreamWriter sw, Logger logger, bool recalculate = false, long game = -1) - { - // If we're supposed to recalculate the statistics, do so - if (recalculate) +"); + + // Now write it out to file as well + string line = ""; + switch (statOutputFormat) { - RecalculateStats(); + case StatOutputFormat.CSV: + line = "\"" + FileName + "\"," + + "\"" + Style.GetBytesReadable(TotalSize) + "\"," + + "\"" + (game == -1 ? Files.Count : game) + "\"," + + "\"" + RomCount + "\"," + + "\"" + DiskCount + "\"," + + "\"" + CRCCount + "\"," + + "\"" + MD5Count + "\"," + + "\"" + SHA1Count + "\"," + + "\"" + NodumpCount + "\"\n"; + break; + case StatOutputFormat.HTML: + line = "\t\t\t" + HttpUtility.HtmlEncode(FileName) + "" + + "" + Style.GetBytesReadable(TotalSize) + "" + + "" + (game == -1 ? Files.Count : game) + "" + + "" + RomCount + "" + + "" + DiskCount + "" + + "" + CRCCount + "" + + "" + MD5Count + "" + + "" + SHA1Count + "" + + "" + NodumpCount + "" + + "\n"; + break; + case StatOutputFormat.None: + default: + line = @"For file '" + FileName + @"': +-------------------------------------------------- + Uncompressed size: " + Style.GetBytesReadable(TotalSize) + @" + Games found: " + (game == -1 ? Files.Count : game) + @" + Roms found: " + RomCount + @" + Disks found: " + DiskCount + @" + Roms with CRC: " + CRCCount + @" + Roms with MD5: " + MD5Count + @" + Roms with SHA-1: " + SHA1Count + @" + Roms with Nodump status: " + NodumpCount + @" + +"; + break; + case StatOutputFormat.TSV: + line = "\"" + FileName + "\"\t" + + "\"" + Style.GetBytesReadable(TotalSize) + "\"\t" + + "\"" + (game == -1 ? Files.Count : game) + "\"\t" + + "\"" + RomCount + "\"\t" + + "\"" + DiskCount + "\"\t" + + "\"" + CRCCount + "\"\t" + + "\"" + MD5Count + "\"\t" + + "\"" + SHA1Count + "\"\t" + + "\"" + NodumpCount + "\"\n"; + break; } - BucketByGame(false, true, logger, false); - if (TotalSize < 0) - { - TotalSize = Int64.MaxValue + TotalSize; - } - sw.Write("\t\t\t" + HttpUtility.HtmlEncode(FileName) + "" - + "" + Style.GetBytesReadable(TotalSize) + "" - + "" + (game == -1 ? Files.Count : game) + "" - + "" + RomCount + "" - + "" + DiskCount + "" - + "" + CRCCount + "" - + "" + MD5Count + "" - + "" + SHA1Count + "" - + "" + NodumpCount + "" - + "\n"); + // Output the line to the streamwriter + sw.Write(line); } #endregion @@ -5057,99 +5091,31 @@ namespace SabreTools.Helper /// Output the stats for a list of input dats as files in a human-readable format /// /// List of input files and folders + /// Name of the output file /// True if single DAT stats are output, false otherwise + /// Set the statistics output format to use /// Logger object for file and console output - /// Logger object for file and console output (statistics) - public static void OutputStats(List inputs, bool single, Logger logger, Logger statLogger) + public static void OutputStats(List inputs, string reportName, bool single, StatOutputFormat statOutputFormat, Logger logger) { - // Make sure we have all files - List newinputs = new List(); - foreach (string input in inputs) + string reportExtension = ""; + switch (statOutputFormat) { - if (File.Exists(input)) - { - newinputs.Add(input); - } - if (Directory.Exists(input)) - { - foreach (string file in Directory.GetFiles(input, "*", SearchOption.AllDirectories)) - { - newinputs.Add(file); - } - } + case StatOutputFormat.CSV: + reportExtension = ".csv"; + break; + case StatOutputFormat.HTML: + reportExtension = ".html"; + break; + case StatOutputFormat.None: + default: + reportExtension = ".txt"; + break; + case StatOutputFormat.TSV: + reportExtension = ".csv"; + break; } - - // Init all total variables - long totalSize = 0; - long totalGame = 0; - long totalRom = 0; - long totalDisk = 0; - long totalCRC = 0; - long totalMD5 = 0; - long totalSHA1 = 0; - long totalNodump = 0; - - /// Now process each of the input files - foreach (string filename in newinputs) - { - logger.Verbose("Beginning stat collection for '" + filename + "'", false); - List games = new List(); - DatFile datdata = new DatFile(); - datdata.Parse(filename, 0, 0, logger); - datdata.BucketByGame(false, true, logger, false); - - // Output single DAT stats (if asked) - if (single) - { - statLogger.User(@"\nFor file '" + filename + @"': ---------------------------------------------------", false); - datdata.OutputStats(logger, statLogger); - } - else - { - logger.User("Adding stats for file '" + filename + "'\n", false); - } - - // Add single DAT stats to totals - totalSize += datdata.TotalSize; - totalGame += datdata.Files.Count; - totalRom += datdata.RomCount; - totalDisk += datdata.DiskCount; - totalCRC += datdata.CRCCount; - totalMD5 += datdata.MD5Count; - totalSHA1 += datdata.SHA1Count; - totalNodump += datdata.NodumpCount; - } - - // Output total DAT stats - if (!single) { logger.User("", false); } - DatFile totaldata = new DatFile - { - TotalSize = totalSize, - RomCount = totalRom, - DiskCount = totalDisk, - CRCCount = totalCRC, - MD5Count = totalMD5, - SHA1Count = totalSHA1, - NodumpCount = totalNodump, - }; - statLogger.User(@"For ALL DATs found ---------------------------------------------------", false); - totaldata.OutputStats(logger, statLogger, game: totalGame); - logger.User(@" -Please check the log folder if the stats scrolled offscreen", false); - } - - /// - /// Output the stats for a list of input dats as files in a human-readable HTML format - /// - /// List of input files and folders - /// True if single DAT stats are output, false otherwise - /// Logger object for file and console output - /// Logger object for file and console output (statistics) - public static void OutputHTMLStats(List inputs, Logger logger) - { - StreamWriter sw = new StreamWriter(File.OpenWrite("report.html")); + reportName += reportExtension; + StreamWriter sw = new StreamWriter(File.OpenWrite(reportName)); // Make sure we have all files List newinputs = new List(); @@ -5168,17 +5134,6 @@ Please check the log folder if the stats scrolled offscreen", false); } } - // Write the HTML header - sw.Write(@" - -
- DAT Statistics Report -
- - - " -+ "\n"); - // Init all total variables long totalSize = 0; long totalGame = 0; @@ -5200,7 +5155,36 @@ Please check the log folder if the stats scrolled offscreen", false); // Output single DAT stats (if asked) logger.User("Adding stats for file '" + filename + "'\n", false); - datdata.OutputHTMLStats(sw, logger); + if (single) + { + string line = ""; + switch (statOutputFormat) + { + case StatOutputFormat.CSV: + line = "\"File Name\",\"Total Size\",\"Games\",\"Roms\",\"Disks\",\"# with CRC\",\"# with MD5\",\"# with SHA-1\",\"Nodumps\"\n"; + break; + case StatOutputFormat.HTML: + line = @" + +
+ DAT Statistics Report +
+ +
File NameTotal SizeGamesRomsDisks# with CRC# with MD5# with SHA-1Nodumps
+ " ++ "\n"; + break; + case StatOutputFormat.None: + default: + break; + case StatOutputFormat.TSV: + line = "\"File Name\"\t\"Total Size\"\t\"Games\"\t\"Roms\"\t\"Disks\"\t\"# with CRC\"\t\"# with MD5\"\t\"# with SHA-1\"\t\"Nodumps\"\n"; + break; + } + sw.Write(line); + + datdata.OutputStats(sw, statOutputFormat, logger); + } // Add single DAT stats to totals totalSize += datdata.TotalSize; @@ -5213,12 +5197,25 @@ Please check the log folder if the stats scrolled offscreen", false); totalNodump += datdata.NodumpCount; } - sw.Write(""); + // Output midpoint separator if needed + string mid = ""; + switch (statOutputFormat) + { + case StatOutputFormat.CSV: + break; + case StatOutputFormat.HTML: + mid = "\n"; + break; + case StatOutputFormat.None: + default: + break; + } + sw.Write(mid); // Output total DAT stats DatFile totaldata = new DatFile { - FileName = "Totals", + FileName = "ALL", TotalSize = totalSize, RomCount = totalRom, DiskCount = totalDisk, @@ -5227,15 +5224,31 @@ Please check the log folder if the stats scrolled offscreen", false); SHA1Count = totalSHA1, NodumpCount = totalNodump, }; - totaldata.OutputHTMLStats(sw, logger, game: totalGame); + totaldata.OutputStats(sw, statOutputFormat, logger, game: totalGame); - // Write HTML footer - sw.Write(@"
File NameTotal SizeGamesRomsDisks# with CRC# with MD5# with SHA-1Nodumps
+ // Output footer if needed + string end = ""; + switch (statOutputFormat) + { + case StatOutputFormat.CSV: + break; + case StatOutputFormat.HTML: + end = @" -"); +"; + break; + case StatOutputFormat.None: + default: + break; + } + sw.Write(end); + sw.Flush(); sw.Dispose(); + + logger.User(@" +Please check the log folder if the stats scrolled offscreen", false); } #endregion diff --git a/SabreTools.Helper/Objects/SimpleSort.cs b/SabreTools.Helper/Objects/SimpleSort.cs index 9e57f406..1325067d 100644 --- a/SabreTools.Helper/Objects/SimpleSort.cs +++ b/SabreTools.Helper/Objects/SimpleSort.cs @@ -262,7 +262,9 @@ namespace SabreTools.Helper _logger.ClearBeneath(Constants.HeaderHeight); Console.SetCursorPosition(0, Constants.HeaderHeight + 1); _logger.User("Stats of the matched ROMs:"); - _matched.OutputStats(_logger, _logger, true); + StreamWriter sw = new StreamWriter(new MemoryStream()); + _matched.OutputStats(sw, StatOutputFormat.None, _logger, true); + sw.Dispose(); // Now output the fixdat based on the original input if asked if (_updateDat) diff --git a/SabreTools.Helper/README.1ST b/SabreTools.Helper/README.1ST index 7e5a4e67..20bf0f3d 100644 --- a/SabreTools.Helper/README.1ST +++ b/SabreTools.Helper/README.1ST @@ -310,18 +310,6 @@ Options: This should only be used if one of the inputs starts with a flag or another already defined input. - -html, --html Get statistics on all input DATs written to HTML - This will output by default the combined statistics for all input DAT files. The stats - that are outputted are as follows: - - Total uncompressed size - - Number of games found - - Number of roms found - - Number of disks found - - Roms that include a CRC - - Roms that include a MD5 - - Roms that include a SHA-1 - - Roms with Nodump status - -st, --stats Get statistics on all input DATs This will output by default the combined statistics for all input DAT files. The stats that are outputted are as follows: @@ -334,11 +322,20 @@ Options: - Roms that include a SHA-1 - Roms with Nodump status + -csv, --csv Write all statistics to CSV + Output all rom information in standardized CSV format + + -html, --html Write all statistics to HTML + This will output by default the combined statistics for all input DAT files. + -si, --single Show individual statistics Optionally, the statistics for each of the individual input DATs can be output as well. This can be useful to show where the size or amount of files found in the combined totals can be broken down from. + -tsv, --tsv Output in Tab-Separated Value format + Output all rom information in standardized TSV format + -ts, --type-split Split a DAT or folder by file types (rom/disk) For a DAT, or set of DATs, allow for splitting based on the types of the files, specifically if the type is a rom or a disk. diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index 0e261312..d767d1a9 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -245,15 +245,6 @@ namespace SabreTools } } - /// - /// Wrap getting statistics on a DAT or folder of DATs to HTML - /// - /// List of inputs to be used - private static void InitHTMLStats(List inputs) - { - DatFile.OutputHTMLStats(inputs, _logger); - } - /// /// Wrap sorting files using an input DAT /// @@ -293,11 +284,10 @@ namespace SabreTools /// /// List of inputs to be used /// True to show individual DAT statistics, false otherwise - private static void InitStats(List inputs, bool single) + /// Set the statistics output format to use + private static void InitStats(List inputs, bool single, StatOutputFormat statOutputFormat) { - Logger statlog = new Logger(true, "stats.txt"); - DatFile.OutputStats(inputs, single, _logger, statlog); - statlog.Close(true); + DatFile.OutputStats(inputs, "report", single, statOutputFormat, _logger); } /// diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index a18fbb96..a80ae7d7 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -62,7 +62,6 @@ namespace SabreTools forceunpack = false, hashsplit = false, headerer = false, - html = false, inplace = false, merge = false, noMD5 = false, @@ -89,6 +88,7 @@ namespace SabreTools slt = -1, seq = -1; OutputFormat outputFormat = 0x0; + StatOutputFormat statOutputFormat = StatOutputFormat.None; string addext = "", author = "", category = "", @@ -159,6 +159,7 @@ namespace SabreTools case "-csv": case "--csv": tsv = false; + statOutputFormat = StatOutputFormat.CSV; break; case "-clean": case "--clean": @@ -215,7 +216,7 @@ namespace SabreTools break; case "-html": case "--html": - html = true; + statOutputFormat = StatOutputFormat.HTML; break; case "-ip": case "--inplace": @@ -324,6 +325,7 @@ namespace SabreTools case "-tsv": case "--tsv": tsv = true; + statOutputFormat = StatOutputFormat.TSV; break; case "-u": case "--unzip": @@ -523,8 +525,8 @@ namespace SabreTools } // If more than one switch is enabled, show the help screen - if (!(extsplit ^ hashsplit ^ headerer ^ html ^ (datfromdir || merge || diffMode != 0 || update - || outputFormat != 0 || tsv != null|| trim) ^ rem ^ stats ^ typesplit)) + if (!(extsplit ^ hashsplit ^ headerer ^ (datfromdir || merge || diffMode != 0 || update + || outputFormat != 0 || trim) ^ rem ^ stats ^ typesplit)) { _logger.Error("Only one feature switch is allowed at a time"); Build.Help(); @@ -533,8 +535,8 @@ namespace SabreTools } // If a switch that requires a filename is set and no file is, show the help screen - if (inputs.Count == 0 && (datfromdir || extsplit || hashsplit || headerer || html - || (merge || diffMode != 0 || update || outputFormat != 0 || tsv != null) || stats || trim || typesplit)) + if (inputs.Count == 0 && (datfromdir || extsplit || hashsplit || headerer + || (merge || diffMode != 0 || update || outputFormat != 0) || stats || trim || typesplit)) { _logger.Error("This feature requires at least one input"); Build.Help(); @@ -569,16 +571,10 @@ namespace SabreTools InitHeaderer(inputs, restore, outDir); } - // Get statistics on input files to HTML - else if (html) - { - InitHTMLStats(inputs); - } - // Get statistics on input files else if (stats) { - InitStats(inputs, single); + InitStats(inputs, single, statOutputFormat); } // Split a DAT by item type