Statistics Collection / Writing Overhaul (#35)

* Add DatStatistics class

* Add isDirectory setting

* Add CalculateStatistics method (nw)

* Add separate stats writing

* Use new methods

* Rename Write -> WriteIndividual

* Naive implementation of new writing (nw)

* Remove unncessary calls

* Make writing more DatFile-like

* Add console flag to constructor

* Remove unused stream constructors

* Move to local writers

* Remove inherent filename

* Fix invocation

* Use SeparatedValueWriter

* Fix final directory stats output

* Use XmlTextWriter for HTML

* Don't output separator on last stat output

* Remove now-completed TODOs

* Remove unused using
This commit is contained in:
Matt Nadareski
2021-02-18 11:13:11 -08:00
committed by GitHub
parent 10d8387883
commit 873431080d
9 changed files with 691 additions and 394 deletions

View File

@@ -33,7 +33,14 @@ namespace RombaSharp.Features
Inputs = new List<string> { Path.GetFullPath(_dats) }; Inputs = new List<string> { Path.GetFullPath(_dats) };
// Now output the stats for all inputs // Now output the stats for all inputs
Statistics.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile); var statistics = Statistics.CalculateStatistics(Inputs, single: true);
Statistics.Write(
statistics,
"rombasharp-datstats",
outDir: null,
baddumpCol: true,
nodumpCol: true,
StatReportFormat.Textfile);
} }
} }
} }

View File

@@ -3,7 +3,9 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles; using SabreTools.DatFiles;
using SabreTools.DatItems; using SabreTools.DatItems;
using SabreTools.IO; using SabreTools.IO;
@@ -28,36 +30,15 @@ namespace SabreTools.DatTools
#endregion #endregion
/// <summary> /// <summary>
/// Output the stats for a list of input dats as files in a human-readable format /// Calculate statistics from a list of inputs
/// </summary> /// </summary>
/// <param name="inputs">List of input files and folders</param> /// <param name="inputs">List of input files and folders</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="single">True if single DAT stats are output, false otherwise</param> /// <param name="single">True if single DAT stats are output, false otherwise</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> public static List<DatStatistics> CalculateStatistics(List<string> inputs, bool single, bool throwOnError = false)
/// <param name="statDatFormat" > Set the statistics output format to use</param>
public static void OutputStats(
List<string> inputs,
string reportName,
string outDir,
bool single,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat)
{ {
// If there's no output format, set the default // Create the output list
if (statDatFormat == StatReportFormat.None) List<DatStatistics> stats = new List<DatStatistics>();
statDatFormat = StatReportFormat.Textfile;
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Get the proper output directory name
outDir = outDir.Ensure();
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outputs = CreateOutStatsNames(outDir, statDatFormat, reportName);
// Make sure we have all files and then order them // Make sure we have all files and then order them
List<ParentablePath> files = PathTool.GetFilesOnly(inputs); List<ParentablePath> files = PathTool.GetFilesOnly(inputs);
@@ -66,99 +47,158 @@ namespace SabreTools.DatTools
.ThenBy(i => Path.GetFileName(i.CurrentPath)) .ThenBy(i => Path.GetFileName(i.CurrentPath))
.ToList(); .ToList();
// Get all of the writers that we need // Init total
List<BaseReport> reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList(); DatStatistics totalStats = new DatStatistics
{
// Write the header, if any Statistics = new ItemDictionary(),
reports.ForEach(report => report.WriteHeader()); DisplayName = "DIR: All DATs",
MachineCount = 0,
// Init all total variables IsDirectory = true,
ItemDictionary totalStats = new ItemDictionary(); };
// Init directory-level variables // Init directory-level variables
string lastdir = null; string lastdir = null;
string basepath = null; DatStatistics dirStats = new DatStatistics
ItemDictionary dirStats = new ItemDictionary(); {
Statistics = new ItemDictionary(),
MachineCount = 0,
IsDirectory = true,
};
// Now process each of the input files // Now process each of the input files
foreach (ParentablePath file in files) foreach (ParentablePath file in files)
{ {
// Get the directory for the current file // Get the directory for the current file
string thisdir = Path.GetDirectoryName(file.CurrentPath); string thisdir = Path.GetDirectoryName(file.CurrentPath);
basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath));
// If we don't have the first file and the directory has changed, show the previous directory stats and reset // If we don't have the first file and the directory has changed, show the previous directory stats and reset
if (lastdir != null && thisdir != lastdir) if (lastdir != null && thisdir != lastdir && single)
{ {
// Output separator if needed dirStats.DisplayName = $"DIR: {WebUtility.HtmlEncode(lastdir)}";
reports.ForEach(report => report.WriteMidSeparator()); dirStats.MachineCount = dirStats.Statistics.GameCount;
stats.Add(dirStats);
DatFile lastdirdat = DatFile.Create(); dirStats = new DatStatistics
{
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); Statistics = new ItemDictionary(),
reports.ForEach(report => report.Write()); MachineCount = 0,
IsDirectory = true,
// Write the mid-footer, if any };
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
} }
logger.Verbose($"Beginning stat collection for '{file.CurrentPath}'"); InternalStopwatch watch = new InternalStopwatch($"Collecting statistics for '{file.CurrentPath}'");
List<string> games = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath, statsOnly: true); List<string> machines = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath, statsOnly: true, throwOnError: throwOnError);
datdata.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); datdata.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Output single DAT stats (if asked) // Add single DAT stats (if asked)
logger.User($"Adding stats for file '{file.CurrentPath}'\n");
if (single) if (single)
{ {
reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items)); DatStatistics individualStats = new DatStatistics
reports.ForEach(report => report.Write()); {
Statistics = datdata.Items,
DisplayName = datdata.Header.FileName,
MachineCount = datdata.Items.Keys.Count,
IsDirectory = false,
};
stats.Add(individualStats);
} }
// Add single DAT stats to dir // Add single DAT stats to dir
dirStats.AddStatistics(datdata.Items); dirStats.Statistics.AddStatistics(datdata.Items);
dirStats.GameCount += datdata.Items.Keys.Count(); dirStats.Statistics.GameCount += datdata.Items.Keys.Count();
// Add single DAT stats to totals // Add single DAT stats to totals
totalStats.AddStatistics(datdata.Items); totalStats.Statistics.AddStatistics(datdata.Items);
totalStats.GameCount += datdata.Items.Keys.Count(); totalStats.Statistics.GameCount += datdata.Items.Keys.Count();
// Make sure to assign the new directory // Make sure to assign the new directory
lastdir = thisdir; lastdir = thisdir;
watch.Stop();
} }
// Output the directory stats one last time // Add last directory stats
reports.ForEach(report => report.WriteMidSeparator());
if (single) if (single)
{ {
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats)); dirStats.DisplayName = $"DIR: {WebUtility.HtmlEncode(lastdir)}";
reports.ForEach(report => report.Write()); dirStats.MachineCount = dirStats.Statistics.GameCount;
stats.Add(dirStats);
} }
// Write the mid-footer, if any // Add total DAT stats
reports.ForEach(report => report.WriteFooterSeparator()); totalStats.MachineCount = totalStats.Statistics.GameCount;
stats.Add(totalStats);
// Write the header, if any return stats;
reports.ForEach(report => report.WriteMidHeader()); }
// Reset the directory stats /// <summary>
dirStats.ResetStatistics(); /// Output the stats for a list of input dats as files in a human-readable format
/// </summary>
/// <param name="stats">List of pre-calculated statistics objects</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat"> Set the statistics output format to use</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the report was written correctly, false otherwise</returns>
public static bool Write(
List<DatStatistics> stats,
string reportName,
string outDir,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat,
bool throwOnError = false)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
{
logger.Verbose("No report format defined, defaulting to textfile");
statDatFormat = StatReportFormat.Textfile;
}
// Output total DAT stats // Get the proper output file name
reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats)); if (string.IsNullOrWhiteSpace(reportName))
reports.ForEach(report => report.Write()); reportName = "report";
// Output footer if needed // Get the proper output directory name
reports.ForEach(report => report.WriteFooter()); outDir = outDir.Ensure();
logger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen"); InternalStopwatch watch = new InternalStopwatch($"Writing out report data to '{outDir}'");
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outfiles = CreateOutStatsNames(outDir, statDatFormat, reportName);
try
{
// Write out all required formats
Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, reportFormat =>
{
string outfile = outfiles[reportFormat];
try
{
BaseReport.Create(reportFormat, stats)?.WriteToFile(outfile, baddumpCol, nodumpCol, throwOnError);
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex, $"Report '{outfile}' could not be written out");
}
});
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
} }
/// <summary> /// <summary>

View File

@@ -116,8 +116,18 @@ namespace SabreTools.DatTools
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
var consoleOutput = BaseReport.Create(StatReportFormat.None, null, true, true); var statsList = new List<DatStatistics>
consoleOutput.ReplaceStatistics(datFile.Header.FileName, datFile.Items.Keys.Count(), datFile.Items); {
new DatStatistics
{
Statistics = datFile.Items,
DisplayName = datFile.Header.FileName,
MachineCount = datFile.Items.Keys.Count(),
IsDirectory = false,
},
};
var consoleOutput = BaseReport.Create(StatReportFormat.None, statsList);
consoleOutput.WriteToFile(null, true, true);
} }
/// <summary> /// <summary>

View File

@@ -1,7 +1,6 @@
using System; using System.Collections.Generic;
using System.IO;
using SabreTools.DatFiles; using SabreTools.Logging;
using SabreTools.Reports.Formats; using SabreTools.Reports.Formats;
namespace SabreTools.Reports namespace SabreTools.Reports
@@ -9,110 +8,57 @@ namespace SabreTools.Reports
/// <summary> /// <summary>
/// Base class for a report output format /// Base class for a report output format
/// </summary> /// </summary>
/// TODO: Can this be overhauled to have all types write like DatFiles?
public abstract class BaseReport public abstract class BaseReport
{ {
protected string _name; #region Logging
protected long _machineCount;
protected ItemDictionary _stats;
protected StreamWriter _writer; /// <summary>
protected bool _baddumpCol; /// Logging object
protected bool _nodumpCol; /// </summary>
protected readonly Logger logger = new Logger();
#endregion
public List<DatStatistics> Statistics { get; set; }
/// <summary> /// <summary>
/// Create a new report from the filename /// Create a new report from the filename
/// </summary> /// </summary>
/// <param name="filename">Name of the file to write out to</param> /// <param name="statsList">List of statistics objects to set</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> public BaseReport(List<DatStatistics> statsList)
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public BaseReport(string filename, bool baddumpCol = false, bool nodumpCol = false)
{ {
var fs = File.Create(filename); Statistics = statsList;
if (fs != null)
_writer = new StreamWriter(fs) { AutoFlush = true };
_baddumpCol = baddumpCol;
_nodumpCol = nodumpCol;
}
/// <summary>
/// Create a new report from the stream
/// </summary>
/// <param name="stream">Output stream to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public BaseReport(Stream stream, bool baddumpCol = false, bool nodumpCol = false)
{
if (!stream.CanWrite)
throw new ArgumentException(nameof(stream));
_writer = new StreamWriter(stream) { AutoFlush = true };
_baddumpCol = baddumpCol;
_nodumpCol = nodumpCol;
} }
/// <summary> /// <summary>
/// Create a specific type of BaseReport to be used based on a format and user inputs /// Create a specific type of BaseReport to be used based on a format and user inputs
/// </summary> /// </summary>
/// <param name="statReportFormat">Format of the Statistics Report to be created</param> /// <param name="statReportFormat">Format of the Statistics Report to be created</param>
/// <param name="filename">Name of the file to write out to</param> /// <param name="statsList">List of statistics objects to set</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <returns>BaseReport of the specific internal type that corresponds to the inputs</returns> /// <returns>BaseReport of the specific internal type that corresponds to the inputs</returns>
public static BaseReport Create(StatReportFormat statReportFormat, string filename, bool baddumpCol, bool nodumpCol) public static BaseReport Create(StatReportFormat statReportFormat, List<DatStatistics> statsList)
{ {
return statReportFormat switch return statReportFormat switch
{ {
StatReportFormat.None => new Textfile(Console.OpenStandardOutput(), baddumpCol, nodumpCol), StatReportFormat.None => new Textfile(statsList, true),
StatReportFormat.Textfile => new Textfile(filename, baddumpCol, nodumpCol), StatReportFormat.Textfile => new Textfile(statsList, false),
StatReportFormat.CSV => new SeparatedValue(filename, ',', baddumpCol, nodumpCol), StatReportFormat.CSV => new SeparatedValue(statsList, ','),
StatReportFormat.HTML => new Html(filename, baddumpCol, nodumpCol), StatReportFormat.HTML => new Html(statsList),
StatReportFormat.SSV => new SeparatedValue(filename, ';', baddumpCol, nodumpCol), StatReportFormat.SSV => new SeparatedValue(statsList, ';'),
StatReportFormat.TSV => new SeparatedValue(filename, '\t', baddumpCol, nodumpCol), StatReportFormat.TSV => new SeparatedValue(statsList, '\t'),
_ => null, _ => null,
}; };
} }
/// <summary> /// <summary>
/// Replace the statistics that is being output /// Create and open an output file for writing direct from a set of statistics
/// </summary> /// </summary>
public void ReplaceStatistics(string datName, long machineCount, ItemDictionary datStats) /// <param name="outfile">Name of the file to write to</param>
{ /// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
_name = datName; /// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
_machineCount = machineCount; /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
_stats = datStats; /// <returns>True if the report was written correctly, false otherwise</returns>
} public abstract bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false);
/// <summary>
/// Write the report to the output stream
/// </summary>
public abstract void Write();
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public abstract void WriteHeader();
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// </summary>
public abstract void WriteMidHeader();
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public abstract void WriteMidSeparator();
/// <summary>
/// Write out the footer-separator to the stream, if any exists
/// </summary>
public abstract void WriteFooterSeparator();
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public abstract void WriteFooter();
/// <summary> /// <summary>
/// Returns the human-readable file size for an arbitrary, 64-bit file size /// Returns the human-readable file size for an arbitrary, 64-bit file size

View File

@@ -0,0 +1,30 @@
using SabreTools.DatFiles;
namespace SabreTools.Reports
{
/// <summary>
/// Statistics wrapper for outputting
/// </summary>
public class DatStatistics
{
/// <summary>
/// ItemDictionary representing the statistics
/// </summary>
public ItemDictionary Statistics { get; set; }
/// <summary>
/// Name to display on output
/// </summary>
public string DisplayName { get; set; }
/// <summary>
/// Total machine count to use on output
/// </summary>
public long MachineCount { get; set; }
/// <summary>
/// Determines if statistics are for a directory or not
/// </summary>
public bool IsDirectory { get; set; } = false;
}
}

View File

@@ -1,6 +1,11 @@
using System; using System;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Net; using System.Net;
using System.Text;
using System.Xml;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats namespace SabreTools.Reports.Formats
{ {
@@ -13,133 +18,314 @@ namespace SabreTools.Reports.Formats
/// <summary> /// <summary>
/// Create a new report from the filename /// Create a new report from the filename
/// </summary> /// </summary>
/// <param name="filename">Name of the file to write out to</param> /// <param name="statsList">List of statistics objects to set</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> public Html(List<DatStatistics> statsList)
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> : base(statsList)
public Html(string filename, bool baddumpCol = false, bool nodumpCol = false)
: base(filename, baddumpCol, nodumpCol)
{ {
} }
/// <summary> /// <inheritdoc/>
/// Create a new report from the stream public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
/// </summary>
/// <param name="datfile">DatFile to write out statistics for</param>
/// <param name="stream">Output stream to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Html(Stream stream, bool baddumpCol = false, bool nodumpCol = false)
: base(stream, baddumpCol, nodumpCol)
{ {
} InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
/// <summary> try
/// Write the report to file {
/// </summary> // Try to create the output file
public override void Write() FileStream fs = File.Create(outfile);
{ if (fs == null)
string line = "\t\t\t<tr" + (_name.StartsWith("DIR: ") {
? $" class=\"dir\"><td>{WebUtility.HtmlEncode(_name.Remove(0, 5))}" logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
: $"><td>{WebUtility.HtmlEncode(_name)}") + "</td>" return false;
+ $"<td align=\"right\">{GetBytesReadable(_stats.TotalSize)}</td>" }
+ $"<td align=\"right\">{_machineCount}</td>"
+ $"<td align=\"right\">{_stats.RomCount}</td>" XmlTextWriter xtw = new XmlTextWriter(fs, Encoding.UTF8)
+ $"<td align=\"right\">{_stats.DiskCount}</td>" {
+ $"<td align=\"right\">{_stats.CRCCount}</td>" Formatting = Formatting.Indented,
+ $"<td align=\"right\">{_stats.MD5Count}</td>" IndentChar = '\t',
+ $"<td align=\"right\">{_stats.SHA1Count}</td>" Indentation = 1
+ $"<td align=\"right\">{_stats.SHA256Count}</td>" };
+ (_baddumpCol ? $"<td align=\"right\">{_stats.BaddumpCount}</td>" : string.Empty)
+ (_nodumpCol ? $"<td align=\"right\">{_stats.NodumpCount}</td>" : string.Empty) // Write out the header
+ "</tr>\n"; WriteHeader(xtw, baddumpCol, nodumpCol);
_writer.Write(line);
_writer.Flush(); // Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteMidSeparator(xtw, baddumpCol, nodumpCol);
WriteIndividual(xtw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
{
WriteFooterSeparator(xtw, baddumpCol, nodumpCol);
WriteMidHeader(xtw, baddumpCol, nodumpCol);
}
}
// If we have a normal statistic
else
{
WriteIndividual(xtw, stat, baddumpCol, nodumpCol);
}
}
WriteFooter(xtw);
xtw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
} }
/// <summary> /// <summary>
/// Write out the header to the stream, if any exists /// Write out the header to the stream, if any exists
/// </summary> /// </summary>
public override void WriteHeader() /// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteHeader(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{ {
_writer.Write(@"<!DOCTYPE html> xtw.WriteDocType("html", null, null, null);
<html> xtw.WriteStartElement("html");
<header>
<title>DAT Statistics Report</title> xtw.WriteStartElement("header");
<style> xtw.WriteElementString("title", "DAT Statistics Report");
body { xtw.WriteElementString("style", @"
background-color: lightgray; body {
} background-color: lightgray;
.dir { }
color: #0088FF; .dir {
} color: #0088FF;
.right { }");
align: right; xtw.WriteEndElement(); // header
}
</style> xtw.WriteStartElement("body");
</header>
<body> xtw.WriteElementString("h2", $"DAT Statistics Report ({DateTime.Now.ToShortDateString()})");
<h2>DAT Statistics Report (" + DateTime.Now.ToShortDateString() + @")</h2>
<table border=string.Empty1string.Empty cellpadding=string.Empty5string.Empty cellspacing=string.Empty0string.Empty> xtw.WriteStartElement("table");
"); xtw.WriteAttributeString("border", "1");
_writer.Flush(); xtw.WriteAttributeString("cellpadding", "5");
xtw.WriteAttributeString("cellspacing", "0");
xtw.Flush();
// Now write the mid header for those who need it // Now write the mid header for those who need it
WriteMidHeader(); WriteMidHeader(xtw, baddumpCol, nodumpCol);
} }
/// <summary> /// <summary>
/// Write out the mid-header to the stream, if any exists /// Write out the mid-header to the stream, if any exists
/// </summary> /// </summary>
public override void WriteMidHeader() /// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteMidHeader(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{ {
_writer.Write(@" <tr bgcolor=string.Emptygraystring.Empty><th>File Name</th><th align=string.Emptyrightstring.Empty>Total Size</th><th align=string.Emptyrightstring.Empty>Games</th><th align=string.Emptyrightstring.Empty>Roms</th>" xtw.WriteStartElement("tr");
+ @"<th align=string.Emptyrightstring.Empty>Disks</th><th align=string.Emptyrightstring.Empty>&#35; with CRC</th><th align=string.Emptyrightstring.Empty>&#35; with MD5</th><th align=string.Emptyrightstring.Empty>&#35; with SHA-1</th><th align=string.Emptyrightstring.Empty>&#35; with SHA-256</th>" xtw.WriteAttributeString("bgcolor", "gray");
+ (_baddumpCol ? "<th class=\".right\">Baddumps</th>" : string.Empty) + (_nodumpCol ? "<th class=\".right\">Nodumps</th>" : string.Empty) + "</tr>\n");
_writer.Flush(); xtw.WriteElementString("th", "File Name");
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Total Size");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Games");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Roms");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Disks");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with CRC");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with MD5");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with SHA-1");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with SHA-256");
xtw.WriteEndElement(); // th
if (baddumpCol)
{
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Baddumps");
xtw.WriteEndElement(); // th
}
if (nodumpCol)
{
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Nodumps");
xtw.WriteEndElement(); // th
}
xtw.WriteEndElement(); // tr
xtw.Flush();
}
/// <summary>
/// Write a single set of statistics
/// </summary>
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteIndividual(XmlTextWriter xtw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
{
bool isDirectory = stat.DisplayName.StartsWith("DIR: ");
xtw.WriteStartElement("tr");
if (isDirectory)
xtw.WriteAttributeString("class", "dir");
xtw.WriteElementString("td", isDirectory ? WebUtility.HtmlEncode(stat.DisplayName.Remove(0, 5)) : WebUtility.HtmlEncode(stat.DisplayName));
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(GetBytesReadable(stat.Statistics.TotalSize));
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.MachineCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.RomCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.DiskCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.CRCCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.MD5Count.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.SHA1Count.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.SHA256Count.ToString());
xtw.WriteEndElement(); // td
if (baddumpCol)
{
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.BaddumpCount.ToString());
xtw.WriteEndElement(); // td
}
if (nodumpCol)
{
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.NodumpCount.ToString());
xtw.WriteEndElement(); // td
}
xtw.WriteEndElement(); // tr
xtw.Flush();
} }
/// <summary> /// <summary>
/// Write out the separator to the stream, if any exists /// Write out the separator to the stream, if any exists
/// </summary> /// </summary>
public override void WriteMidSeparator() /// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteMidSeparator(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{ {
_writer.Write("<tr><td colspan=\"" xtw.WriteStartElement("tr");
+ (_baddumpCol && _nodumpCol
? "12" xtw.WriteStartElement("td");
: (_baddumpCol ^ _nodumpCol xtw.WriteAttributeString("colspan", baddumpCol && nodumpCol ? "12" : (baddumpCol ^ nodumpCol ? "11" : "10"));
? "11" xtw.WriteEndElement(); // td
: "10")
) xtw.WriteEndElement(); // tr
+ "\"></td></tr>\n"); xtw.Flush();
_writer.Flush();
} }
/// <summary> /// <summary>
/// Write out the footer-separator to the stream, if any exists /// Write out the footer-separator to the stream, if any exists
/// </summary> /// </summary>
public override void WriteFooterSeparator() /// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteFooterSeparator(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{ {
_writer.Write("<tr border=\"0\"><td colspan=\"" xtw.WriteStartElement("tr");
+ (_baddumpCol && _nodumpCol xtw.WriteAttributeString("border", "0");
? "12"
: (_baddumpCol ^ _nodumpCol xtw.WriteStartElement("td");
? "11" xtw.WriteAttributeString("colspan", baddumpCol && nodumpCol ? "12" : (baddumpCol ^ nodumpCol ? "11" : "10"));
: "10") xtw.WriteEndElement(); // td
)
+ "\"></td></tr>\n"); xtw.WriteEndElement(); // tr
_writer.Flush(); xtw.Flush();
} }
/// <summary> /// <summary>
/// Write out the footer to the stream, if any exists /// Write out the footer to the stream, if any exists
/// </summary> /// </summary>
public override void WriteFooter() /// <param name="xtw">XmlTextWriter to write to</param>
private void WriteFooter(XmlTextWriter xtw)
{ {
_writer.Write(@" </table> xtw.WriteEndElement(); // table
</body> xtw.WriteEndElement(); // body
</html> xtw.WriteEndElement(); // html
"); xtw.Flush();
_writer.Flush();
} }
} }
} }

View File

@@ -1,4 +1,10 @@
using System.IO; using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using SabreTools.IO.Writers;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats namespace SabreTools.Reports.Formats
{ {
@@ -12,94 +18,142 @@ namespace SabreTools.Reports.Formats
/// <summary> /// <summary>
/// Create a new report from the filename /// Create a new report from the filename
/// </summary> /// </summary>
/// <param name="filename">Name of the file to write out to</param> /// <param name="statsList">List of statistics objects to set</param>
/// <param name="separator">Separator character to use in output</param> /// <param name="separator">Separator character to use in output</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> public SeparatedValue(List<DatStatistics> statsList, char separator)
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> : base(statsList)
public SeparatedValue(string filename, char separator, bool baddumpCol = false, bool nodumpCol = false)
: base(filename, baddumpCol, nodumpCol)
{ {
_separator = separator; _separator = separator;
} }
/// <summary> /// <inheritdoc/>
/// Create a new report from the input DatFile and the stream public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
/// </summary>
/// <param name="stream">Output stream to write to</param>
/// <param name="separator">Separator character to use in output</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public SeparatedValue(Stream stream, char separator, bool baddumpCol = false, bool nodumpCol = false)
: base(stream, baddumpCol, nodumpCol)
{ {
_separator = separator; InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
}
/// <summary> try
/// Write the report to file {
/// </summary> // Try to create the output file
public override void Write() FileStream fs = File.Create(outfile);
{ if (fs == null)
string line = string.Format("\"" + _name + "\"{0}" {
+ "\"" + _stats.TotalSize + "\"{0}" logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
+ "\"" + _machineCount + "\"{0}" return false;
+ "\"" + _stats.RomCount + "\"{0}" }
+ "\"" + _stats.DiskCount + "\"{0}"
+ "\"" + _stats.CRCCount + "\"{0}"
+ "\"" + _stats.MD5Count + "\"{0}"
+ "\"" + _stats.SHA1Count + "\"{0}"
+ "\"" + _stats.SHA256Count + "\"{0}"
+ "\"" + _stats.SHA384Count + "\"{0}"
+ "\"" + _stats.SHA512Count + "\""
+ (_baddumpCol ? "{0}\"" + _stats.BaddumpCount + "\"" : string.Empty)
+ (_nodumpCol ? "{0}\"" + _stats.NodumpCount + "\"" : string.Empty)
+ "\n", _separator);
_writer.Write(line); SeparatedValueWriter svw = new SeparatedValueWriter(fs, Encoding.UTF8)
_writer.Flush(); {
Separator = _separator,
Quotes = true,
};
// Write out the header
WriteHeader(svw, baddumpCol, nodumpCol);
// Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteIndividual(svw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
WriteFooterSeparator(svw);
}
// If we have a normal statistic
else
{
WriteIndividual(svw, stat, baddumpCol, nodumpCol);
}
}
svw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
} }
/// <summary> /// <summary>
/// Write out the header to the stream, if any exists /// Write out the header to the stream, if any exists
/// </summary> /// </summary>
public override void WriteHeader() /// <param name="svw">SeparatedValueWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteHeader(SeparatedValueWriter svw, bool baddumpCol, bool nodumpCol)
{ {
_writer.Write(string.Format("\"File Name\"{0}\"Total Size\"{0}\"Games\"{0}\"Roms\"{0}\"Disks\"{0}\"# with CRC\"{0}\"# with MD5\"{0}\"# with SHA-1\"{0}\"# with SHA-256\"" string[] headers = new string[]
+ (_baddumpCol ? "{0}\"BadDumps\"" : string.Empty) + (_nodumpCol ? "{0}\"Nodumps\"" : string.Empty) + "\n", _separator)); {
_writer.Flush(); "File Name",
"Total Size",
"Games",
"Roms",
"Disks",
"# with CRC",
"# with MD5",
"# with SHA-1",
"# with SHA-256",
"# with SHA-384",
"# with SHA-512",
baddumpCol ? "BadDumps" : string.Empty,
nodumpCol ? "Nodumps" : string.Empty,
};
svw.WriteHeader(headers);
svw.Flush();
} }
/// <summary> /// <summary>
/// Write out the mid-header to the stream, if any exists /// Write a single set of statistics
/// </summary> /// </summary>
public override void WriteMidHeader() /// <param name="svw">SeparatedValueWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteIndividual(SeparatedValueWriter svw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
{ {
// This call is a no-op for separated value formats string[] values = new string[]
} {
stat.DisplayName,
/// <summary> stat.Statistics.TotalSize.ToString(),
/// Write out the separator to the stream, if any exists stat.MachineCount.ToString(),
/// </summary> stat.Statistics.RomCount.ToString(),
public override void WriteMidSeparator() stat.Statistics.DiskCount.ToString(),
{ stat.Statistics.CRCCount.ToString(),
// This call is a no-op for separated value formats stat.Statistics.MD5Count.ToString(),
stat.Statistics.SHA1Count.ToString(),
stat.Statistics.SHA256Count.ToString(),
stat.Statistics.SHA384Count.ToString(),
stat.Statistics.SHA512Count.ToString(),
baddumpCol ? stat.Statistics.BaddumpCount.ToString() : string.Empty,
nodumpCol ? stat.Statistics.NodumpCount.ToString() : string.Empty,
};
svw.WriteValues(values);
svw.Flush();
} }
/// <summary> /// <summary>
/// Write out the footer-separator to the stream, if any exists /// Write out the footer-separator to the stream, if any exists
/// </summary> /// </summary>
public override void WriteFooterSeparator() /// <param name="svw">SeparatedValueWriter to write to</param>
private void WriteFooterSeparator(SeparatedValueWriter svw)
{ {
_writer.Write("\n"); svw.WriteString("\n");
_writer.Flush(); svw.Flush();
}
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public override void WriteFooter()
{
// This call is a no-op for separated value formats
} }
} }
} }

View File

@@ -1,4 +1,8 @@
using System.IO; using System;
using System.Collections.Generic;
using System.IO;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats namespace SabreTools.Reports.Formats
{ {
@@ -7,98 +11,118 @@ namespace SabreTools.Reports.Formats
/// </summary> /// </summary>
internal class Textfile : BaseReport internal class Textfile : BaseReport
{ {
private readonly bool _writeToConsole;
/// <summary> /// <summary>
/// Create a new report from the filename /// Create a new report from the filename
/// </summary> /// </summary>
/// <param name="filename">Name of the file to write out to</param> /// <param name="statsList">List of statistics objects to set</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> /// <param name="writeToConsole">True to write to consoke output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> public Textfile(List<DatStatistics> statsList, bool writeToConsole)
public Textfile(string filename, bool baddumpCol = false, bool nodumpCol = false) : base(statsList)
: base(filename, baddumpCol, nodumpCol)
{ {
_writeToConsole = writeToConsole;
}
/// <inheritdoc/>
public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
{
InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
try
{
// Try to create the output file
Stream fs = _writeToConsole ? Console.OpenStandardOutput() : File.Create(outfile);
if (fs == null)
{
logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
return false;
}
StreamWriter sw = new StreamWriter(fs);
// Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteIndividual(sw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
WriteFooterSeparator(sw);
}
// If we have a normal statistic
else
{
WriteIndividual(sw, stat, baddumpCol, nodumpCol);
}
}
sw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
} }
/// <summary> /// <summary>
/// Create a new report from the stream /// Write a single set of statistics
/// </summary> /// </summary>
/// <param name="stream">Output stream to write to</param> /// <param name="sw">StreamWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> /// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> /// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Textfile(Stream stream, bool baddumpCol = false, bool nodumpCol = false) private void WriteIndividual(StreamWriter sw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
: base(stream, baddumpCol, nodumpCol)
{ {
} string line = @"'" + stat.DisplayName + @"':
/// <summary>
/// Write the report to file
/// </summary>
public override void Write()
{
string line = @"'" + _name + @"':
-------------------------------------------------- --------------------------------------------------
Uncompressed size: " + GetBytesReadable(_stats.TotalSize) + @" Uncompressed size: " + GetBytesReadable(stat.Statistics.TotalSize) + @"
Games found: " + _machineCount + @" Games found: " + stat.MachineCount + @"
Roms found: " + _stats.RomCount + @" Roms found: " + stat.Statistics.RomCount + @"
Disks found: " + _stats.DiskCount + @" Disks found: " + stat.Statistics.DiskCount + @"
Roms with CRC: " + _stats.CRCCount + @" Roms with CRC: " + stat.Statistics.CRCCount + @"
Roms with MD5: " + _stats.MD5Count + @" Roms with MD5: " + stat.Statistics.MD5Count + @"
Roms with SHA-1: " + _stats.SHA1Count + @" Roms with SHA-1: " + stat.Statistics.SHA1Count + @"
Roms with SHA-256: " + _stats.SHA256Count + @" Roms with SHA-256: " + stat.Statistics.SHA256Count + @"
Roms with SHA-384: " + _stats.SHA384Count + @" Roms with SHA-384: " + stat.Statistics.SHA384Count + @"
Roms with SHA-512: " + _stats.SHA512Count + "\n"; Roms with SHA-512: " + stat.Statistics.SHA512Count + "\n";
if (_baddumpCol) if (baddumpCol)
line += " Roms with BadDump status: " + _stats.BaddumpCount + "\n"; line += " Roms with BadDump status: " + stat.Statistics.BaddumpCount + "\n";
if (_nodumpCol) if (nodumpCol)
line += " Roms with Nodump status: " + _stats.NodumpCount + "\n"; line += " Roms with Nodump status: " + stat.Statistics.NodumpCount + "\n";
// For spacing between DATs // For spacing between DATs
line += "\n\n"; line += "\n\n";
_writer.Write(line); sw.Write(line);
_writer.Flush(); sw.Flush();
}
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public override void WriteHeader()
{
// This call is a no-op for textfile output
}
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// </summary>
public override void WriteMidHeader()
{
// This call is a no-op for textfile output
}
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public override void WriteMidSeparator()
{
// This call is a no-op for textfile output
} }
/// <summary> /// <summary>
/// Write out the footer-separator to the stream, if any exists /// Write out the footer-separator to the stream, if any exists
/// </summary> /// </summary>
public override void WriteFooterSeparator() /// <param name="sw">StreamWriter to write to</param>
private void WriteFooterSeparator(StreamWriter sw)
{ {
_writer.Write("\n"); sw.Write("\n");
_writer.Flush(); sw.Flush();
}
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public override void WriteFooter()
{
// This call is a no-op for textfile output
} }
} }
} }

View File

@@ -58,11 +58,11 @@ The stats that are outputted are as follows:
filename = Path.GetFileName(filename); filename = Path.GetFileName(filename);
} }
Statistics.OutputStats( var statistics = Statistics.CalculateStatistics(Inputs, GetBoolean(features, IndividualValue));
Inputs, Statistics.Write(
statistics,
filename, filename,
OutputDir, OutputDir,
GetBoolean(features, IndividualValue),
GetBoolean(features, BaddumpColumnValue), GetBoolean(features, BaddumpColumnValue),
GetBoolean(features, NodumpColumnValue), GetBoolean(features, NodumpColumnValue),
GetStatReportFormat(features)); GetStatReportFormat(features));