[Stats] Misc. cleanup and Stats overhaul

SabreTools.Helper/Data/Build.cs

@@ -143,9 +143,11 @@ namespace SabreTools.Helper
 					helptext.Add("	-out=				Output directory");
 					helptext.Add("  -hs, --hash-split	Split a DAT or folder by best-available hashes");
 					helptext.Add("	-out=			Output directory");
-					helptext.Add("  -html, --html		Write statistics on all input DATs to HTML");
 					helptext.Add("  -st, --stats		Get statistics on all input DATs");
+					helptext.Add("	-csv, --csv			Output in Comma-Separated Value format");
+					helptext.Add("	-html, --html		Write stats to HTML");
 					helptext.Add("	-si, --single		Show individual statistics");
+					helptext.Add("	-tsv, --tsv			Output in Tab-Separated Value format");
 					helptext.Add("  -ts, --type-split		Split a DAT or folder by file types (rom/disk)");
 					helptext.Add("	-out=			Output directory");
 					helptext.Add("  -ud, --update		Update a DAT file");

SabreTools.Helper/Data/Enums.cs

@@ -1,26 +1,5 @@
 namespace SabreTools.Helper
 {
-	#region DATabase
-
-	/// <summary>
-	/// Possible DAT import classes
-	/// </summary>
-	public enum DatType
-	{
-		none = 0,
-		Custom,
-		MAME,
-		NoIntro,
-		Redump,
-		TOSEC,
-		TruRip,
-		NonGood,
-		MaybeIntro,
-		Good,
-	}
-
-	#endregion
-
 	#region DatFile related
 
 	/// <summary>
@@ -97,6 +76,17 @@
 		NotNodump = 5, // This is a fake flag that is used for filter only
 	}
 
+	/// <summary>
+	/// Determine which format to output Stats to
+	/// </summary>
+	public enum StatOutputFormat
+	{
+		None = 0,
+		HTML = 1,
+		CSV = 2,
+		TSV = 3,
+	}
+
 	#endregion
 
 	#region Skippers and Mappers

SabreTools.Helper/Objects/Dat/DatFile.cs

@@ -3003,7 +3003,9 @@ namespace SabreTools.Helper
 			// Output initial statistics, for kicks
 			if (stats)
 			{
-				OutputStats(logger, logger, (RomCount + DiskCount == 0));
+				StreamWriter sw = new StreamWriter(new MemoryStream());
+				OutputStats(sw, StatOutputFormat.None, logger, (RomCount + DiskCount == 0));
+				sw.Dispose();
 			}
 
 			// Bucket roms by game name and optionally dedupe
@@ -4335,11 +4337,12 @@ namespace SabreTools.Helper
 		/// <summary>
 		/// Output the stats for the Dat in a human-readable format
 		/// </summary>
+		/// <param name="sw">StreamWriter representing the output file or stream for the statistics</param>
+		/// <param name="statOutputFormat">Set the statistics output format to use</param>
 		/// <param name="logger">Logger object for file and console writing</param>
-		/// <param name="statLogger">Logger object for file and console output (statistics)</param>
 		/// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param>
 		/// <param name="game">Number of games to use, -1 means recalculate games (default)</param>
-		public void OutputStats(Logger logger, Logger statLogger, bool recalculate = false, long game = -1)
+		public void OutputStats(StreamWriter sw, StatOutputFormat statOutputFormat, Logger logger, bool recalculate = false, long game = -1)
 		{
 			// If we're supposed to recalculate the statistics, do so
 			if (recalculate)
@@ -4352,7 +4355,11 @@ namespace SabreTools.Helper
 			{
 				TotalSize = Int64.MaxValue + TotalSize;
 			}
-			statLogger.User("    Uncompressed size:       " + Style.GetBytesReadable(TotalSize) + @"
+
+			// Log the results to screen
+			logger.User(@"For file '" + FileName + @"':
+--------------------------------------------------
+    Uncompressed size:       " + Style.GetBytesReadable(TotalSize) + @"
     Games found:             " + (game == -1 ? Files.Count : game) + @"
     Roms found:              " + RomCount + @"
     Disks found:             " + DiskCount + @"
@@ -4360,30 +4367,26 @@ namespace SabreTools.Helper
     Roms with MD5:           " + MD5Count + @"
     Roms with SHA-1:         " + SHA1Count + @"
     Roms with Nodump status: " + NodumpCount + @"
-", false);
-		}
 
-		/// <summary>
-		/// Output the stats for the Dat in a human-readable HTML format
-		/// </summary>
-		/// <param name="sw">StreamWriter representing the output file or stream for the statistics</param>
-		/// <param name="logger">Logger object for file and console writing</param>
-		/// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param>
-		/// <param name="game">Number of games to use, -1 means recalculate games (default)</param>
-		public void OutputHTMLStats(StreamWriter sw, Logger logger, bool recalculate = false, long game = -1)
-		{
-			// If we're supposed to recalculate the statistics, do so
-			if (recalculate)
-			{
-				RecalculateStats();
-			}
+");
 
-			BucketByGame(false, true, logger, false);
-			if (TotalSize < 0)
+			// Now write it out to file as well
+			string line = "";
+			switch (statOutputFormat)
 			{
-				TotalSize = Int64.MaxValue + TotalSize;
-			}
-			sw.Write("\t\t\t<tr><td>" + HttpUtility.HtmlEncode(FileName) + "</td>"
+				case StatOutputFormat.CSV:
+					line = "\"" + FileName + "\","
+						+ "\"" + Style.GetBytesReadable(TotalSize) + "\","
+						+ "\"" + (game == -1 ? Files.Count : game) + "\","
+						+ "\"" + RomCount + "\","
+						+ "\"" + DiskCount + "\","
+						+ "\"" + CRCCount + "\","
+						+ "\"" + MD5Count + "\","
+						+ "\"" + SHA1Count + "\","
+						+ "\"" + NodumpCount + "\"\n";
+					break;
+				case StatOutputFormat.HTML:
+					line = "\t\t\t<tr><td>" + HttpUtility.HtmlEncode(FileName) + "</td>"
 						+ "<td>" + Style.GetBytesReadable(TotalSize) + "</td>"
 						+ "<td>" + (game == -1 ? Files.Count : game) + "</td>"
 						+ "<td>" + RomCount + "</td>"
@@ -4392,7 +4395,38 @@ namespace SabreTools.Helper
 						+ "<td>" + MD5Count + "</td>"
 						+ "<td>" + SHA1Count + "</td>"
 						+ "<td>" + NodumpCount + "</td>"
-				+ "</tr>\n");
+						+ "</tr>\n";
+					break;
+				case StatOutputFormat.None:
+				default:
+					line = @"For file '" + FileName + @"':
+--------------------------------------------------
+    Uncompressed size:       " + Style.GetBytesReadable(TotalSize) + @"
+    Games found:             " + (game == -1 ? Files.Count : game) + @"
+    Roms found:              " + RomCount + @"
+    Disks found:             " + DiskCount + @"
+    Roms with CRC:           " + CRCCount + @"
+    Roms with MD5:           " + MD5Count + @"
+    Roms with SHA-1:         " + SHA1Count + @"
+    Roms with Nodump status: " + NodumpCount + @"
+
+";
+					break;
+				case StatOutputFormat.TSV:
+					line = "\"" + FileName + "\"\t"
+						+ "\"" + Style.GetBytesReadable(TotalSize) + "\"\t"
+						+ "\"" + (game == -1 ? Files.Count : game) + "\"\t"
+						+ "\"" + RomCount + "\"\t"
+						+ "\"" + DiskCount + "\"\t"
+						+ "\"" + CRCCount + "\"\t"
+						+ "\"" + MD5Count + "\"\t"
+						+ "\"" + SHA1Count + "\"\t"
+						+ "\"" + NodumpCount + "\"\n";
+					break;
+			}
+
+			// Output the line to the streamwriter
+			sw.Write(line);
 		}
 
 		#endregion
@@ -5057,11 +5091,32 @@ namespace SabreTools.Helper
 		/// Output the stats for a list of input dats as files in a human-readable format
 		/// </summary>
 		/// <param name="inputs">List of input files and folders</param>
+		/// <param name="reportName">Name of the output file</param>
 		/// <param name="single">True if single DAT stats are output, false otherwise</param>
+		/// <param name="statOutputFormat" > Set the statistics output format to use</param>
 		/// <param name="logger">Logger object for file and console output</param>
-		/// <param name="statLogger">Logger object for file and console output (statistics)</param>
-		public static void OutputStats(List<string> inputs, bool single, Logger logger, Logger statLogger)
+		public static void OutputStats(List<string> inputs, string reportName, bool single, StatOutputFormat statOutputFormat, Logger logger)
 		{
+			string reportExtension = "";
+			switch (statOutputFormat)
+			{
+				case StatOutputFormat.CSV:
+					reportExtension = ".csv";
+					break;
+				case StatOutputFormat.HTML:
+					reportExtension = ".html";
+					break;
+				case StatOutputFormat.None:
+				default:
+					reportExtension = ".txt";
+					break;
+				case StatOutputFormat.TSV:
+					reportExtension = ".csv";
+					break;
+			}
+			reportName += reportExtension;
+			StreamWriter sw = new StreamWriter(File.OpenWrite(reportName));
+
 			// Make sure we have all files
 			List<string> newinputs = new List<string>();
 			foreach (string input in inputs)
@@ -5099,108 +5154,37 @@ namespace SabreTools.Helper
 				datdata.BucketByGame(false, true, logger, false);
 
 				// Output single DAT stats (if asked)
+				logger.User("Adding stats for file '" + filename + "'\n", false);
 				if (single)
 				{
-					statLogger.User(@"\nFor file '" + filename + @"':
---------------------------------------------------", false);
-					datdata.OutputStats(logger, statLogger);
-				}
-				else
+					string line = "";
+					switch (statOutputFormat)
 					{
-					logger.User("Adding stats for file '" + filename + "'\n", false);
-				}
-
-				// Add single DAT stats to totals
-				totalSize += datdata.TotalSize;
-				totalGame += datdata.Files.Count;
-				totalRom += datdata.RomCount;
-				totalDisk += datdata.DiskCount;
-				totalCRC += datdata.CRCCount;
-				totalMD5 += datdata.MD5Count;
-				totalSHA1 += datdata.SHA1Count;
-				totalNodump += datdata.NodumpCount;
-			}
-
-			// Output total DAT stats
-			if (!single) { logger.User("", false); }
-			DatFile totaldata = new DatFile
-			{
-				TotalSize = totalSize,
-				RomCount = totalRom,
-				DiskCount = totalDisk,
-				CRCCount = totalCRC,
-				MD5Count = totalMD5,
-				SHA1Count = totalSHA1,
-				NodumpCount = totalNodump,
-			};
-			statLogger.User(@"For ALL DATs found
---------------------------------------------------", false);
-			totaldata.OutputStats(logger, statLogger, game: totalGame);
-			logger.User(@"
-Please check the log folder if the stats scrolled offscreen", false);
-		}
-
-		/// <summary>
-		/// Output the stats for a list of input dats as files in a human-readable HTML format
-		/// </summary>
-		/// <param name="inputs">List of input files and folders</param>
-		/// <param name="single">True if single DAT stats are output, false otherwise</param>
-		/// <param name="logger">Logger object for file and console output</param>
-		/// <param name="statLogger">Logger object for file and console output (statistics)</param>
-		public static void OutputHTMLStats(List<string> inputs, Logger logger)
-		{
-			StreamWriter sw = new StreamWriter(File.OpenWrite("report.html"));
-
-			// Make sure we have all files
-			List<string> newinputs = new List<string>();
-			foreach (string input in inputs)
-			{
-				if (File.Exists(input))
-				{
-					newinputs.Add(input);
-				}
-				if (Directory.Exists(input))
-				{
-					foreach (string file in Directory.GetFiles(input, "*", SearchOption.AllDirectories))
-					{
-						newinputs.Add(file);
-					}
-				}
-			}
-
-			// Write the HTML header
-			sw.Write(@"<!DOCTYPE html>
+						case StatOutputFormat.CSV:
+							line = "\"File Name\",\"Total Size\",\"Games\",\"Roms\",\"Disks\",\"# with CRC\",\"# with MD5\",\"# with SHA-1\",\"Nodumps\"\n";
+							break;
+						case StatOutputFormat.HTML:
+							line = @"<!DOCTYPE html>
 <html>
 	<header>
 		<title>DAT Statistics Report</title>
 	</header>
 	<body>
-		<table border=""1"" cellpadding=""0"" cellspacing=""0"">
+		<table border=""1"" cellpadding=""5"" cellspacing=""0"">
 			<tr><th>File Name</th><th>Total Size</th><th>Games</th><th>Roms</th><th>Disks</th><th>&#35; with CRC</th>"
-+ "<th>&#35; with MD5</th><th>&#35; with SHA-1</th><th>Nodumps</th></tr>\n");
++ "<th>&#35; with MD5</th><th>&#35; with SHA-1</th><th>Nodumps</th></tr>\n";
+							break;
+						case StatOutputFormat.None:
+						default:
+							break;
+						case StatOutputFormat.TSV:
+							line = "\"File Name\"\t\"Total Size\"\t\"Games\"\t\"Roms\"\t\"Disks\"\t\"# with CRC\"\t\"# with MD5\"\t\"# with SHA-1\"\t\"Nodumps\"\n";
+							break;
+					}
+					sw.Write(line);
 
-			// Init all total variables
-			long totalSize = 0;
-			long totalGame = 0;
-			long totalRom = 0;
-			long totalDisk = 0;
-			long totalCRC = 0;
-			long totalMD5 = 0;
-			long totalSHA1 = 0;
-			long totalNodump = 0;
-
-			/// Now process each of the input files
-			foreach (string filename in newinputs)
-			{
-				logger.Verbose("Beginning stat collection for '" + filename + "'", false);
-				List<string> games = new List<string>();
-				DatFile datdata = new DatFile();
-				datdata.Parse(filename, 0, 0, logger);
-				datdata.BucketByGame(false, true, logger, false);
-
-				// Output single DAT stats (if asked)
-				logger.User("Adding stats for file '" + filename + "'\n", false);
-				datdata.OutputHTMLStats(sw, logger);
+					datdata.OutputStats(sw, statOutputFormat, logger);
+				}
 
 				// Add single DAT stats to totals
 				totalSize += datdata.TotalSize;
@@ -5213,12 +5197,25 @@ Please check the log folder if the stats scrolled offscreen", false);
 				totalNodump += datdata.NodumpCount;
 			}
 
-			sw.Write("<tr><td colspan=\"9\"></td></tr>");
+			// Output midpoint separator if needed
+			string mid = "";
+			switch (statOutputFormat)
+			{
+				case StatOutputFormat.CSV:
+					break;
+				case StatOutputFormat.HTML:
+					mid = "<tr><td colspan=\"9\"></td></tr>\n";
+					break;
+				case StatOutputFormat.None:
+				default:
+					break;
+			}
+			sw.Write(mid);
 
 			// Output total DAT stats
 			DatFile totaldata = new DatFile
 			{
-				FileName = "Totals",
+				FileName = "ALL",
 				TotalSize = totalSize,
 				RomCount = totalRom,
 				DiskCount = totalDisk,
@@ -5227,15 +5224,31 @@ Please check the log folder if the stats scrolled offscreen", false);
 				SHA1Count = totalSHA1,
 				NodumpCount = totalNodump,
 			};
-			totaldata.OutputHTMLStats(sw, logger, game: totalGame);
+			totaldata.OutputStats(sw, statOutputFormat, logger, game: totalGame);
 
-			// Write HTML footer
-			sw.Write(@"		</table>
+			// Output footer if needed
+			string end = "";
+			switch (statOutputFormat)
+			{
+				case StatOutputFormat.CSV:
+					break;
+				case StatOutputFormat.HTML:
+					end = @"		</table>
 	</body>
 </html>
-");
+";
+					break;
+				case StatOutputFormat.None:
+				default:
+					break;
+			}
+			sw.Write(end);
+
 			sw.Flush();
 			sw.Dispose();
+
+			logger.User(@"
+Please check the log folder if the stats scrolled offscreen", false);
 		}
 
 		#endregion

SabreTools.Helper/Objects/SimpleSort.cs

@@ -262,7 +262,9 @@ namespace SabreTools.Helper
 			_logger.ClearBeneath(Constants.HeaderHeight);
 			Console.SetCursorPosition(0, Constants.HeaderHeight + 1);
 			_logger.User("Stats of the matched ROMs:");
-			_matched.OutputStats(_logger, _logger, true);
+			StreamWriter sw = new StreamWriter(new MemoryStream());
+			_matched.OutputStats(sw, StatOutputFormat.None, _logger, true);
+			sw.Dispose();
 
 			// Now output the fixdat based on the original input if asked
 			if (_updateDat)

SabreTools.Helper/README.1ST

@@ -310,18 +310,6 @@ Options:
 		This should only be used if one of the inputs starts with a flag or another already
 		defined input.
 		
-	-html, --html		Get statistics on all input DATs written to HTML
-		This will output by default the combined statistics for all input DAT files. The stats
-		that are outputted are as follows:
-			- Total uncompressed size
-			- Number of games found
-			- Number of roms found
-			- Number of disks found
-			- Roms that include a CRC
-			- Roms that include a MD5
-			- Roms that include a SHA-1
-			- Roms with Nodump status
-		
 	-st, --stats		Get statistics on all input DATs
 		This will output by default the combined statistics for all input DAT files. The stats
 		that are outputted are as follows:
@@ -334,11 +322,20 @@ Options:
 			- Roms that include a SHA-1
 			- Roms with Nodump status
 			
+		-csv, --csv			Write all statistics to CSV
+			Output all rom information in standardized CSV format
+			
+		-html, --html		Write all statistics to HTML
+			This will output by default the combined statistics for all input DAT files.
+			
 		-si, --single		Show individual statistics
 			Optionally, the statistics for each of the individual input DATs can be output
 			as well. This can be useful to show where the size or amount of files found
 			in the combined totals can be broken down from.
 			
+		-tsv, --tsv			Output in Tab-Separated Value format
+			Output all rom information in standardized TSV format
+			
 	-ts, --type-split		Split a DAT or folder by file types (rom/disk)
 		For a DAT, or set of DATs, allow for splitting based on the types of the files,
 		specifically if the type is a rom or a disk.

SabreTools/Partials/SabreTools_Inits.cs

@@ -245,15 +245,6 @@ namespace SabreTools
 			}
 		}
 
-		/// <summary>
-		/// Wrap getting statistics on a DAT or folder of DATs to HTML
-		/// </summary>
-		/// <param name="inputs">List of inputs to be used</param>
-		private static void InitHTMLStats(List<string> inputs)
-		{
-			DatFile.OutputHTMLStats(inputs, _logger);
-		}
-
 		/// <summary>
 		/// Wrap sorting files using an input DAT
 		/// </summary>
@@ -293,11 +284,10 @@ namespace SabreTools
 		/// </summary>
 		/// <param name="inputs">List of inputs to be used</param>
 		/// <param name="single">True to show individual DAT statistics, false otherwise</param>
-		private static void InitStats(List<string> inputs, bool single)
+		/// <param name="statOutputFormat">Set the statistics output format to use</param>
+		private static void InitStats(List<string> inputs, bool single, StatOutputFormat statOutputFormat)
 		{
-			Logger statlog = new Logger(true, "stats.txt");
-			DatFile.OutputStats(inputs, single, _logger, statlog);
-			statlog.Close(true);
+			DatFile.OutputStats(inputs, "report", single, statOutputFormat, _logger);
 		}
 
 		/// <summary>

SabreTools/SabreTools.cs

@@ -62,7 +62,6 @@ namespace SabreTools
 				forceunpack = false,
 				hashsplit = false,
 				headerer = false,
-				html = false,
 				inplace = false,
 				merge = false,
 				noMD5 = false,
@@ -89,6 +88,7 @@ namespace SabreTools
 				slt = -1,
 				seq = -1;
 			OutputFormat outputFormat = 0x0;
+			StatOutputFormat statOutputFormat = StatOutputFormat.None;
 			string addext = "",
 				author = "",
 				category = "",
@@ -159,6 +159,7 @@ namespace SabreTools
 					case "-csv":
 					case "--csv":
 						tsv = false;
+						statOutputFormat = StatOutputFormat.CSV;
 						break;
 					case "-clean":
 					case "--clean":
@@ -215,7 +216,7 @@ namespace SabreTools
 						break;
 					case "-html":
 					case "--html":
-						html = true;
+						statOutputFormat = StatOutputFormat.HTML;
 						break;
 					case "-ip":
 					case "--inplace":
@@ -324,6 +325,7 @@ namespace SabreTools
 					case "-tsv":
 					case "--tsv":
 						tsv = true;
+						statOutputFormat = StatOutputFormat.TSV;
 						break;
 					case "-u":
 					case "--unzip":
@@ -523,8 +525,8 @@ namespace SabreTools
 			}
 
 			// If more than one switch is enabled, show the help screen
-			if (!(extsplit ^ hashsplit ^ headerer ^ html ^ (datfromdir || merge || diffMode != 0 || update
-				|| outputFormat != 0 || tsv != null|| trim) ^ rem ^ stats ^ typesplit))
+			if (!(extsplit ^ hashsplit ^ headerer ^ (datfromdir || merge || diffMode != 0 || update
+				|| outputFormat != 0 || trim) ^ rem ^ stats ^ typesplit))
 			{
 				_logger.Error("Only one feature switch is allowed at a time");
 				Build.Help();
@@ -533,8 +535,8 @@ namespace SabreTools
 			}
 
 			// If a switch that requires a filename is set and no file is, show the help screen
-			if (inputs.Count == 0 && (datfromdir || extsplit || hashsplit || headerer || html
-				|| (merge || diffMode != 0 || update || outputFormat != 0 || tsv != null) || stats || trim || typesplit))
+			if (inputs.Count == 0 && (datfromdir || extsplit || hashsplit || headerer
+				|| (merge || diffMode != 0 || update || outputFormat != 0) || stats || trim || typesplit))
 			{
 				_logger.Error("This feature requires at least one input");
 				Build.Help();
@@ -569,16 +571,10 @@ namespace SabreTools
 				InitHeaderer(inputs, restore, outDir);
 			}
 
-			// Get statistics on input files to HTML
-			else if (html)
-			{
-				InitHTMLStats(inputs);
-			}
-
 			// Get statistics on input files
 			else if (stats)
 			{
-				InitStats(inputs, single);
+				InitStats(inputs, single, statOutputFormat);
 			}
 
 			// Split a DAT by item type