claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Statistics Collection / Writing Overhaul (#35)

Browse Source 
* Add DatStatistics class

* Add isDirectory setting

* Add CalculateStatistics method (nw)

* Add separate stats writing

* Use new methods

* Rename Write -> WriteIndividual

* Naive implementation of new writing (nw)

* Remove unncessary calls

* Make writing more DatFile-like

* Add console flag to constructor

* Remove unused stream constructors

* Move to local writers

* Remove inherent filename

* Fix invocation

* Use SeparatedValueWriter

* Fix final directory stats output

* Use XmlTextWriter for HTML

* Don't output separator on last stat output

* Remove now-completed TODOs

* Remove unused using
This commit is contained in:
Matt Nadareski
2021-02-18 11:13:11 -08:00
committed by GitHub
parent 10d8387883
commit 873431080d
9 changed files with 691 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
RombaSharp/Features/DatStats.cs
View File

@@ -33,7 +33,14 @@ namespace RombaSharp.Features
Inputs = new List<string> { Path.GetFullPath(_dats) };
// Now output the stats for all inputs
Statistics.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile);
var statistics = Statistics.CalculateStatistics(Inputs, single: true);
Statistics.Write(
statistics,
"rombasharp-datstats",
outDir: null,
baddumpCol: true,
nodumpCol: true,
StatReportFormat.Textfile);
}
}
}

204
SabreTools.DatTools/Statistics.cs
View File

@@ -3,7 +3,9 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.IO;
@@ -28,36 +30,15 @@ namespace SabreTools.DatTools
#endregion
/// <summary>
/// Output the stats for a list of input dats as files in a human-readable format
/// Calculate statistics from a list of inputs
/// </summary>
/// <param name="inputs">List of input files and folders</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="single">True if single DAT stats are output, false otherwise</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat" > Set the statistics output format to use</param>
public static void OutputStats(
List<string> inputs,
string reportName,
string outDir,
bool single,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat)
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static List<DatStatistics> CalculateStatistics(List<string> inputs, bool single, bool throwOnError = false)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
statDatFormat = StatReportFormat.Textfile;
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Get the proper output directory name
outDir = outDir.Ensure();
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outputs = CreateOutStatsNames(outDir, statDatFormat, reportName);
// Create the output list
List<DatStatistics> stats = new List<DatStatistics>();
// Make sure we have all files and then order them
List<ParentablePath> files = PathTool.GetFilesOnly(inputs);
@@ -66,99 +47,158 @@ namespace SabreTools.DatTools
.ThenBy(i => Path.GetFileName(i.CurrentPath))
.ToList();
// Get all of the writers that we need
List<BaseReport> reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList();
// Write the header, if any
reports.ForEach(report => report.WriteHeader());
// Init all total variables
ItemDictionary totalStats = new ItemDictionary();
// Init total
DatStatistics totalStats = new DatStatistics
{
Statistics = new ItemDictionary(),
DisplayName = "DIR: All DATs",
MachineCount = 0,
IsDirectory = true,
};
// Init directory-level variables
string lastdir = null;
string basepath = null;
ItemDictionary dirStats = new ItemDictionary();
DatStatistics dirStats = new DatStatistics
{
Statistics = new ItemDictionary(),
MachineCount = 0,
IsDirectory = true,
};
// Now process each of the input files
foreach (ParentablePath file in files)
{
// Get the directory for the current file
string thisdir = Path.GetDirectoryName(file.CurrentPath);
basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath));
// If we don't have the first file and the directory has changed, show the previous directory stats and reset
if (lastdir != null && thisdir != lastdir)
if (lastdir != null && thisdir != lastdir && single)
{
// Output separator if needed
reports.ForEach(report => report.WriteMidSeparator());
DatFile lastdirdat = DatFile.Create();
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
dirStats.DisplayName = $"DIR: {WebUtility.HtmlEncode(lastdir)}";
dirStats.MachineCount = dirStats.Statistics.GameCount;
stats.Add(dirStats);
dirStats = new DatStatistics
{
Statistics = new ItemDictionary(),
MachineCount = 0,
IsDirectory = true,
};
}
logger.Verbose($"Beginning stat collection for '{file.CurrentPath}'");
List<string> games = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath, statsOnly: true);
InternalStopwatch watch = new InternalStopwatch($"Collecting statistics for '{file.CurrentPath}'");
List<string> machines = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath, statsOnly: true, throwOnError: throwOnError);
datdata.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
logger.User($"Adding stats for file '{file.CurrentPath}'\n");
// Add single DAT stats (if asked)
if (single)
{
reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items));
reports.ForEach(report => report.Write());
DatStatistics individualStats = new DatStatistics
{
Statistics = datdata.Items,
DisplayName = datdata.Header.FileName,
MachineCount = datdata.Items.Keys.Count,
IsDirectory = false,
};
stats.Add(individualStats);
}
// Add single DAT stats to dir
dirStats.AddStatistics(datdata.Items);
dirStats.GameCount += datdata.Items.Keys.Count();
dirStats.Statistics.AddStatistics(datdata.Items);
dirStats.Statistics.GameCount += datdata.Items.Keys.Count();
// Add single DAT stats to totals
totalStats.AddStatistics(datdata.Items);
totalStats.GameCount += datdata.Items.Keys.Count();
totalStats.Statistics.AddStatistics(datdata.Items);
totalStats.Statistics.GameCount += datdata.Items.Keys.Count();
// Make sure to assign the new directory
lastdir = thisdir;
watch.Stop();
}
// Output the directory stats one last time
reports.ForEach(report => report.WriteMidSeparator());
// Add last directory stats
if (single)
{
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
dirStats.DisplayName = $"DIR: {WebUtility.HtmlEncode(lastdir)}";
dirStats.MachineCount = dirStats.Statistics.GameCount;
stats.Add(dirStats);
}
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Add total DAT stats
totalStats.MachineCount = totalStats.Statistics.GameCount;
stats.Add(totalStats);
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
return stats;
}
// Reset the directory stats
dirStats.ResetStatistics();
/// <summary>
/// Output the stats for a list of input dats as files in a human-readable format
/// </summary>
/// <param name="stats">List of pre-calculated statistics objects</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat"> Set the statistics output format to use</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the report was written correctly, false otherwise</returns>
public static bool Write(
List<DatStatistics> stats,
string reportName,
string outDir,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat,
bool throwOnError = false)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
{
logger.Verbose("No report format defined, defaulting to textfile");
statDatFormat = StatReportFormat.Textfile;
}
// Output total DAT stats
reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats));
reports.ForEach(report => report.Write());
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Output footer if needed
reports.ForEach(report => report.WriteFooter());
// Get the proper output directory name
outDir = outDir.Ensure();
logger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen");
InternalStopwatch watch = new InternalStopwatch($"Writing out report data to '{outDir}'");
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outfiles = CreateOutStatsNames(outDir, statDatFormat, reportName);
try
{
// Write out all required formats
Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, reportFormat =>
{
string outfile = outfiles[reportFormat];
try
{
BaseReport.Create(reportFormat, stats)?.WriteToFile(outfile, baddumpCol, nodumpCol, throwOnError);
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex, $"Report '{outfile}' could not be written out");
}
});
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
}
/// <summary>

14
SabreTools.DatTools/Writer.cs
View File

@@ -116,8 +116,18 @@ namespace SabreTools.DatTools
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
var consoleOutput = BaseReport.Create(StatReportFormat.None, null, true, true);
consoleOutput.ReplaceStatistics(datFile.Header.FileName, datFile.Items.Keys.Count(), datFile.Items);
var statsList = new List<DatStatistics>
{
new DatStatistics
{
Statistics = datFile.Items,
DisplayName = datFile.Header.FileName,
MachineCount = datFile.Items.Keys.Count(),
IsDirectory = false,
},
};
var consoleOutput = BaseReport.Create(StatReportFormat.None, statsList);
consoleOutput.WriteToFile(null, true, true);
}
/// <summary>

112
SabreTools.Reports/BaseReport.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.IO;
using System.Collections.Generic;
using SabreTools.DatFiles;
using SabreTools.Logging;
using SabreTools.Reports.Formats;
namespace SabreTools.Reports
@@ -9,110 +8,57 @@ namespace SabreTools.Reports
/// <summary>
/// Base class for a report output format
/// </summary>
/// TODO: Can this be overhauled to have all types write like DatFiles?
public abstract class BaseReport
{
protected string _name;
protected long _machineCount;
protected ItemDictionary _stats;
#region Logging
protected StreamWriter _writer;
protected bool _baddumpCol;
protected bool _nodumpCol;
/// <summary>
/// Logging object
/// </summary>
protected readonly Logger logger = new Logger();
#endregion
public List<DatStatistics> Statistics { get; set; }
/// <summary>
/// Create a new report from the filename
/// </summary>
/// <param name="filename">Name of the file to write out to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public BaseReport(string filename, bool baddumpCol = false, bool nodumpCol = false)
/// <param name="statsList">List of statistics objects to set</param>
public BaseReport(List<DatStatistics> statsList)
{
var fs = File.Create(filename);
if (fs != null)
_writer = new StreamWriter(fs) { AutoFlush = true };
_baddumpCol = baddumpCol;
_nodumpCol = nodumpCol;
}
/// <summary>
/// Create a new report from the stream
/// </summary>
/// <param name="stream">Output stream to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public BaseReport(Stream stream, bool baddumpCol = false, bool nodumpCol = false)
{
if (!stream.CanWrite)
throw new ArgumentException(nameof(stream));
_writer = new StreamWriter(stream) { AutoFlush = true };
_baddumpCol = baddumpCol;
_nodumpCol = nodumpCol;
Statistics = statsList;
}
/// <summary>
/// Create a specific type of BaseReport to be used based on a format and user inputs
/// </summary>
/// <param name="statReportFormat">Format of the Statistics Report to be created</param>
/// <param name="filename">Name of the file to write out to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statsList">List of statistics objects to set</param>
/// <returns>BaseReport of the specific internal type that corresponds to the inputs</returns>
public static BaseReport Create(StatReportFormat statReportFormat, string filename, bool baddumpCol, bool nodumpCol)
public static BaseReport Create(StatReportFormat statReportFormat, List<DatStatistics> statsList)
{
return statReportFormat switch
{
StatReportFormat.None => new Textfile(Console.OpenStandardOutput(), baddumpCol, nodumpCol),
StatReportFormat.Textfile => new Textfile(filename, baddumpCol, nodumpCol),
StatReportFormat.CSV => new SeparatedValue(filename, ',', baddumpCol, nodumpCol),
StatReportFormat.HTML => new Html(filename, baddumpCol, nodumpCol),
StatReportFormat.SSV => new SeparatedValue(filename, ';', baddumpCol, nodumpCol),
StatReportFormat.TSV => new SeparatedValue(filename, '\t', baddumpCol, nodumpCol),
StatReportFormat.None => new Textfile(statsList, true),
StatReportFormat.Textfile => new Textfile(statsList, false),
StatReportFormat.CSV => new SeparatedValue(statsList, ','),
StatReportFormat.HTML => new Html(statsList),
StatReportFormat.SSV => new SeparatedValue(statsList, ';'),
StatReportFormat.TSV => new SeparatedValue(statsList, '\t'),
_ => null,
};
}
/// <summary>
/// Replace the statistics that is being output
/// Create and open an output file for writing direct from a set of statistics
/// </summary>
public void ReplaceStatistics(string datName, long machineCount, ItemDictionary datStats)
{
_name = datName;
_machineCount = machineCount;
_stats = datStats;
}
/// <summary>
/// Write the report to the output stream
/// </summary>
public abstract void Write();
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public abstract void WriteHeader();
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// </summary>
public abstract void WriteMidHeader();
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public abstract void WriteMidSeparator();
/// <summary>
/// Write out the footer-separator to the stream, if any exists
/// </summary>
public abstract void WriteFooterSeparator();
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public abstract void WriteFooter();
/// <param name="outfile">Name of the file to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the report was written correctly, false otherwise</returns>
public abstract bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false);
/// <summary>
/// Returns the human-readable file size for an arbitrary, 64-bit file size

30
SabreTools.Reports/DatStatistics.cs Normal file
View File

@@ -0,0 +1,30 @@
using SabreTools.DatFiles;
namespace SabreTools.Reports
{
/// <summary>
/// Statistics wrapper for outputting
/// </summary>
public class DatStatistics
{
/// <summary>
/// ItemDictionary representing the statistics
/// </summary>
public ItemDictionary Statistics { get; set; }
/// <summary>
/// Name to display on output
/// </summary>
public string DisplayName { get; set; }
/// <summary>
/// Total machine count to use on output
/// </summary>
public long MachineCount { get; set; }
/// <summary>
/// Determines if statistics are for a directory or not
/// </summary>
public bool IsDirectory { get; set; } = false;
}
}

366
SabreTools.Reports/Formats/Html.cs
View File

@@ -1,6 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Xml;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats
{
@@ -13,133 +18,314 @@ namespace SabreTools.Reports.Formats
/// <summary>
/// Create a new report from the filename
/// </summary>
/// <param name="filename">Name of the file to write out to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Html(string filename, bool baddumpCol = false, bool nodumpCol = false)
: base(filename, baddumpCol, nodumpCol)
/// <param name="statsList">List of statistics objects to set</param>
public Html(List<DatStatistics> statsList)
: base(statsList)
{
}
/// <summary>
/// Create a new report from the stream
/// </summary>
/// <param name="datfile">DatFile to write out statistics for</param>
/// <param name="stream">Output stream to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Html(Stream stream, bool baddumpCol = false, bool nodumpCol = false)
: base(stream, baddumpCol, nodumpCol)
/// <inheritdoc/>
public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
{
}
InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
/// <summary>
/// Write the report to file
/// </summary>
public override void Write()
{
string line = "\t\t\t<tr" + (_name.StartsWith("DIR: ")
? $" class=\"dir\"><td>{WebUtility.HtmlEncode(_name.Remove(0, 5))}"
: $"><td>{WebUtility.HtmlEncode(_name)}") + "</td>"
+ $"<td align=\"right\">{GetBytesReadable(_stats.TotalSize)}</td>"
+ $"<td align=\"right\">{_machineCount}</td>"
+ $"<td align=\"right\">{_stats.RomCount}</td>"
+ $"<td align=\"right\">{_stats.DiskCount}</td>"
+ $"<td align=\"right\">{_stats.CRCCount}</td>"
+ $"<td align=\"right\">{_stats.MD5Count}</td>"
+ $"<td align=\"right\">{_stats.SHA1Count}</td>"
+ $"<td align=\"right\">{_stats.SHA256Count}</td>"
+ (_baddumpCol ? $"<td align=\"right\">{_stats.BaddumpCount}</td>" : string.Empty)
+ (_nodumpCol ? $"<td align=\"right\">{_stats.NodumpCount}</td>" : string.Empty)
+ "</tr>\n";
_writer.Write(line);
_writer.Flush();
try
{
// Try to create the output file
FileStream fs = File.Create(outfile);
if (fs == null)
{
logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
return false;
}
XmlTextWriter xtw = new XmlTextWriter(fs, Encoding.UTF8)
{
Formatting = Formatting.Indented,
IndentChar = '\t',
Indentation = 1
};
// Write out the header
WriteHeader(xtw, baddumpCol, nodumpCol);
// Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteMidSeparator(xtw, baddumpCol, nodumpCol);
WriteIndividual(xtw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
{
WriteFooterSeparator(xtw, baddumpCol, nodumpCol);
WriteMidHeader(xtw, baddumpCol, nodumpCol);
}
}
// If we have a normal statistic
else
{
WriteIndividual(xtw, stat, baddumpCol, nodumpCol);
}
}
WriteFooter(xtw);
xtw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
}
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public override void WriteHeader()
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteHeader(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{
_writer.Write(@"<!DOCTYPE html>
<html>
<header>
<title>DAT Statistics Report</title>
<style>
body {
background-color: lightgray;
}
.dir {
color: #0088FF;
}
.right {
align: right;
}
</style>
</header>
<body>
<h2>DAT Statistics Report (" + DateTime.Now.ToShortDateString() + @")</h2>
<table border=string.Empty1string.Empty cellpadding=string.Empty5string.Empty cellspacing=string.Empty0string.Empty>
");
_writer.Flush();
xtw.WriteDocType("html", null, null, null);
xtw.WriteStartElement("html");
xtw.WriteStartElement("header");
xtw.WriteElementString("title", "DAT Statistics Report");
xtw.WriteElementString("style", @"
body {
background-color: lightgray;
}
.dir {
color: #0088FF;
}");
xtw.WriteEndElement(); // header
xtw.WriteStartElement("body");
xtw.WriteElementString("h2", $"DAT Statistics Report ({DateTime.Now.ToShortDateString()})");
xtw.WriteStartElement("table");
xtw.WriteAttributeString("border", "1");
xtw.WriteAttributeString("cellpadding", "5");
xtw.WriteAttributeString("cellspacing", "0");
xtw.Flush();
// Now write the mid header for those who need it
WriteMidHeader();
WriteMidHeader(xtw, baddumpCol, nodumpCol);
}
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// </summary>
public override void WriteMidHeader()
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteMidHeader(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{
_writer.Write(@" <tr bgcolor=string.Emptygraystring.Empty><th>File Name</th><th align=string.Emptyrightstring.Empty>Total Size</th><th align=string.Emptyrightstring.Empty>Games</th><th align=string.Emptyrightstring.Empty>Roms</th>"
+ @"<th align=string.Emptyrightstring.Empty>Disks</th><th align=string.Emptyrightstring.Empty>&#35; with CRC</th><th align=string.Emptyrightstring.Empty>&#35; with MD5</th><th align=string.Emptyrightstring.Empty>&#35; with SHA-1</th><th align=string.Emptyrightstring.Empty>&#35; with SHA-256</th>"
+ (_baddumpCol ? "<th class=\".right\">Baddumps</th>" : string.Empty) + (_nodumpCol ? "<th class=\".right\">Nodumps</th>" : string.Empty) + "</tr>\n");
_writer.Flush();
xtw.WriteStartElement("tr");
xtw.WriteAttributeString("bgcolor", "gray");
xtw.WriteElementString("th", "File Name");
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Total Size");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Games");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Roms");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Disks");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with CRC");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with MD5");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with SHA-1");
xtw.WriteEndElement(); // th
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("# with SHA-256");
xtw.WriteEndElement(); // th
if (baddumpCol)
{
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Baddumps");
xtw.WriteEndElement(); // th
}
if (nodumpCol)
{
xtw.WriteStartElement("th");
xtw.WriteAttributeString("align", "right");
xtw.WriteString("Nodumps");
xtw.WriteEndElement(); // th
}
xtw.WriteEndElement(); // tr
xtw.Flush();
}
/// <summary>
/// Write a single set of statistics
/// </summary>
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteIndividual(XmlTextWriter xtw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
{
bool isDirectory = stat.DisplayName.StartsWith("DIR: ");
xtw.WriteStartElement("tr");
if (isDirectory)
xtw.WriteAttributeString("class", "dir");
xtw.WriteElementString("td", isDirectory ? WebUtility.HtmlEncode(stat.DisplayName.Remove(0, 5)) : WebUtility.HtmlEncode(stat.DisplayName));
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(GetBytesReadable(stat.Statistics.TotalSize));
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.MachineCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.RomCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.DiskCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.CRCCount.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.MD5Count.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.SHA1Count.ToString());
xtw.WriteEndElement(); // td
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.SHA256Count.ToString());
xtw.WriteEndElement(); // td
if (baddumpCol)
{
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.BaddumpCount.ToString());
xtw.WriteEndElement(); // td
}
if (nodumpCol)
{
xtw.WriteStartElement("td");
xtw.WriteAttributeString("align", "right");
xtw.WriteString(stat.Statistics.NodumpCount.ToString());
xtw.WriteEndElement(); // td
}
xtw.WriteEndElement(); // tr
xtw.Flush();
}
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public override void WriteMidSeparator()
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteMidSeparator(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{
_writer.Write("<tr><td colspan=\""
+ (_baddumpCol && _nodumpCol
? "12"
: (_baddumpCol ^ _nodumpCol
? "11"
: "10")
)
+ "\"></td></tr>\n");
_writer.Flush();
xtw.WriteStartElement("tr");
xtw.WriteStartElement("td");
xtw.WriteAttributeString("colspan", baddumpCol && nodumpCol ? "12" : (baddumpCol ^ nodumpCol ? "11" : "10"));
xtw.WriteEndElement(); // td
xtw.WriteEndElement(); // tr
xtw.Flush();
}
/// <summary>
/// Write out the footer-separator to the stream, if any exists
/// </summary>
public override void WriteFooterSeparator()
/// <param name="xtw">XmlTextWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteFooterSeparator(XmlTextWriter xtw, bool baddumpCol, bool nodumpCol)
{
_writer.Write("<tr border=\"0\"><td colspan=\""
+ (_baddumpCol && _nodumpCol
? "12"
: (_baddumpCol ^ _nodumpCol
? "11"
: "10")
)
+ "\"></td></tr>\n");
_writer.Flush();
xtw.WriteStartElement("tr");
xtw.WriteAttributeString("border", "0");
xtw.WriteStartElement("td");
xtw.WriteAttributeString("colspan", baddumpCol && nodumpCol ? "12" : (baddumpCol ^ nodumpCol ? "11" : "10"));
xtw.WriteEndElement(); // td
xtw.WriteEndElement(); // tr
xtw.Flush();
}
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public override void WriteFooter()
/// <param name="xtw">XmlTextWriter to write to</param>
private void WriteFooter(XmlTextWriter xtw)
{
_writer.Write(@" </table>
</body>
</html>
");
_writer.Flush();
xtw.WriteEndElement(); // table
xtw.WriteEndElement(); // body
xtw.WriteEndElement(); // html
xtw.Flush();
}
}
}

182
SabreTools.Reports/Formats/SeparatedValue.cs
View File

@@ -1,4 +1,10 @@
using System.IO;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using SabreTools.IO.Writers;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats
{
@@ -12,94 +18,142 @@ namespace SabreTools.Reports.Formats
/// <summary>
/// Create a new report from the filename
/// </summary>
/// <param name="filename">Name of the file to write out to</param>
/// <param name="statsList">List of statistics objects to set</param>
/// <param name="separator">Separator character to use in output</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public SeparatedValue(string filename, char separator, bool baddumpCol = false, bool nodumpCol = false)
: base(filename, baddumpCol, nodumpCol)
public SeparatedValue(List<DatStatistics> statsList, char separator)
: base(statsList)
{
_separator = separator;
}
/// <summary>
/// Create a new report from the input DatFile and the stream
/// </summary>
/// <param name="stream">Output stream to write to</param>
/// <param name="separator">Separator character to use in output</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public SeparatedValue(Stream stream, char separator, bool baddumpCol = false, bool nodumpCol = false)
: base(stream, baddumpCol, nodumpCol)
/// <inheritdoc/>
public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
{
_separator = separator;
}
InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
/// <summary>
/// Write the report to file
/// </summary>
public override void Write()
{
string line = string.Format("\"" + _name + "\"{0}"
+ "\"" + _stats.TotalSize + "\"{0}"
+ "\"" + _machineCount + "\"{0}"
+ "\"" + _stats.RomCount + "\"{0}"
+ "\"" + _stats.DiskCount + "\"{0}"
+ "\"" + _stats.CRCCount + "\"{0}"
+ "\"" + _stats.MD5Count + "\"{0}"
+ "\"" + _stats.SHA1Count + "\"{0}"
+ "\"" + _stats.SHA256Count + "\"{0}"
+ "\"" + _stats.SHA384Count + "\"{0}"
+ "\"" + _stats.SHA512Count + "\""
+ (_baddumpCol ? "{0}\"" + _stats.BaddumpCount + "\"" : string.Empty)
+ (_nodumpCol ? "{0}\"" + _stats.NodumpCount + "\"" : string.Empty)
+ "\n", _separator);
try
{
// Try to create the output file
FileStream fs = File.Create(outfile);
if (fs == null)
{
logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
return false;
}
_writer.Write(line);
_writer.Flush();
SeparatedValueWriter svw = new SeparatedValueWriter(fs, Encoding.UTF8)
{
Separator = _separator,
Quotes = true,
};
// Write out the header
WriteHeader(svw, baddumpCol, nodumpCol);
// Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteIndividual(svw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
WriteFooterSeparator(svw);
}
// If we have a normal statistic
else
{
WriteIndividual(svw, stat, baddumpCol, nodumpCol);
}
}
svw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
}
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public override void WriteHeader()
/// <param name="svw">SeparatedValueWriter to write to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteHeader(SeparatedValueWriter svw, bool baddumpCol, bool nodumpCol)
{
_writer.Write(string.Format("\"File Name\"{0}\"Total Size\"{0}\"Games\"{0}\"Roms\"{0}\"Disks\"{0}\"# with CRC\"{0}\"# with MD5\"{0}\"# with SHA-1\"{0}\"# with SHA-256\""
+ (_baddumpCol ? "{0}\"BadDumps\"" : string.Empty) + (_nodumpCol ? "{0}\"Nodumps\"" : string.Empty) + "\n", _separator));
_writer.Flush();
string[] headers = new string[]
{
"File Name",
"Total Size",
"Games",
"Roms",
"Disks",
"# with CRC",
"# with MD5",
"# with SHA-1",
"# with SHA-256",
"# with SHA-384",
"# with SHA-512",
baddumpCol ? "BadDumps" : string.Empty,
nodumpCol ? "Nodumps" : string.Empty,
};
svw.WriteHeader(headers);
svw.Flush();
}
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// Write a single set of statistics
/// </summary>
public override void WriteMidHeader()
/// <param name="svw">SeparatedValueWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
private void WriteIndividual(SeparatedValueWriter svw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
{
// This call is a no-op for separated value formats
}
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public override void WriteMidSeparator()
{
// This call is a no-op for separated value formats
string[] values = new string[]
{
stat.DisplayName,
stat.Statistics.TotalSize.ToString(),
stat.MachineCount.ToString(),
stat.Statistics.RomCount.ToString(),
stat.Statistics.DiskCount.ToString(),
stat.Statistics.CRCCount.ToString(),
stat.Statistics.MD5Count.ToString(),
stat.Statistics.SHA1Count.ToString(),
stat.Statistics.SHA256Count.ToString(),
stat.Statistics.SHA384Count.ToString(),
stat.Statistics.SHA512Count.ToString(),
baddumpCol ? stat.Statistics.BaddumpCount.ToString() : string.Empty,
nodumpCol ? stat.Statistics.NodumpCount.ToString() : string.Empty,
};
svw.WriteValues(values);
svw.Flush();
}
/// <summary>
/// Write out the footer-separator to the stream, if any exists
/// </summary>
public override void WriteFooterSeparator()
/// <param name="svw">SeparatedValueWriter to write to</param>
private void WriteFooterSeparator(SeparatedValueWriter svw)
{
_writer.Write("\n");
_writer.Flush();
}
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public override void WriteFooter()
{
// This call is a no-op for separated value formats
svw.WriteString("\n");
svw.Flush();
}
}
}

162
SabreTools.Reports/Formats/Textfile.cs
View File

@@ -1,4 +1,8 @@
using System.IO;
using System;
using System.Collections.Generic;
using System.IO;
using SabreTools.Logging;
namespace SabreTools.Reports.Formats
{
@@ -7,98 +11,118 @@ namespace SabreTools.Reports.Formats
/// </summary>
internal class Textfile : BaseReport
{
private readonly bool _writeToConsole;
/// <summary>
/// Create a new report from the filename
/// </summary>
/// <param name="filename">Name of the file to write out to</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Textfile(string filename, bool baddumpCol = false, bool nodumpCol = false)
: base(filename, baddumpCol, nodumpCol)
/// <param name="statsList">List of statistics objects to set</param>
/// <param name="writeToConsole">True to write to consoke output, false otherwise</param>
public Textfile(List<DatStatistics> statsList, bool writeToConsole)
: base(statsList)
{
_writeToConsole = writeToConsole;
}
/// <inheritdoc/>
public override bool WriteToFile(string outfile, bool baddumpCol, bool nodumpCol, bool throwOnError = false)
{
InternalStopwatch watch = new InternalStopwatch($"Writing statistics to '{outfile}");
try
{
// Try to create the output file
Stream fs = _writeToConsole ? Console.OpenStandardOutput() : File.Create(outfile);
if (fs == null)
{
logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable");
return false;
}
StreamWriter sw = new StreamWriter(fs);
// Now process each of the statistics
for (int i = 0; i < Statistics.Count; i++)
{
// Get the current statistic
DatStatistics stat = Statistics[i];
// If we have a directory statistic
if (stat.IsDirectory)
{
WriteIndividual(sw, stat, baddumpCol, nodumpCol);
// If we have anything but the last value, write the separator
if (i < Statistics.Count - 1)
WriteFooterSeparator(sw);
}
// If we have a normal statistic
else
{
WriteIndividual(sw, stat, baddumpCol, nodumpCol);
}
}
sw.Dispose();
fs.Dispose();
}
catch (Exception ex) when (!throwOnError)
{
logger.Error(ex);
return false;
}
finally
{
watch.Stop();
}
return true;
}
/// <summary>
/// Create a new report from the stream
/// Write a single set of statistics
/// </summary>
/// <param name="stream">Output stream to write to</param>
/// <param name="sw">StreamWriter to write to</param>
/// <param name="stat">DatStatistics object to write out</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
public Textfile(Stream stream, bool baddumpCol = false, bool nodumpCol = false)
: base(stream, baddumpCol, nodumpCol)
private void WriteIndividual(StreamWriter sw, DatStatistics stat, bool baddumpCol, bool nodumpCol)
{
}
/// <summary>
/// Write the report to file
/// </summary>
public override void Write()
{
string line = @"'" + _name + @"':
string line = @"'" + stat.DisplayName + @"':
--------------------------------------------------
Uncompressed size: " + GetBytesReadable(_stats.TotalSize) + @"
Games found: " + _machineCount + @"
Roms found: " + _stats.RomCount + @"
Disks found: " + _stats.DiskCount + @"
Roms with CRC: " + _stats.CRCCount + @"
Roms with MD5: " + _stats.MD5Count + @"
Roms with SHA-1: " + _stats.SHA1Count + @"
Roms with SHA-256: " + _stats.SHA256Count + @"
Roms with SHA-384: " + _stats.SHA384Count + @"
Roms with SHA-512: " + _stats.SHA512Count + "\n";
Uncompressed size: " + GetBytesReadable(stat.Statistics.TotalSize) + @"
Games found: " + stat.MachineCount + @"
Roms found: " + stat.Statistics.RomCount + @"
Disks found: " + stat.Statistics.DiskCount + @"
Roms with CRC: " + stat.Statistics.CRCCount + @"
Roms with MD5: " + stat.Statistics.MD5Count + @"
Roms with SHA-1: " + stat.Statistics.SHA1Count + @"
Roms with SHA-256: " + stat.Statistics.SHA256Count + @"
Roms with SHA-384: " + stat.Statistics.SHA384Count + @"
Roms with SHA-512: " + stat.Statistics.SHA512Count + "\n";
if (_baddumpCol)
line += " Roms with BadDump status: " + _stats.BaddumpCount + "\n";
if (baddumpCol)
line += " Roms with BadDump status: " + stat.Statistics.BaddumpCount + "\n";
if (_nodumpCol)
line += " Roms with Nodump status: " + _stats.NodumpCount + "\n";
if (nodumpCol)
line += " Roms with Nodump status: " + stat.Statistics.NodumpCount + "\n";
// For spacing between DATs
line += "\n\n";
_writer.Write(line);
_writer.Flush();
}
/// <summary>
/// Write out the header to the stream, if any exists
/// </summary>
public override void WriteHeader()
{
// This call is a no-op for textfile output
}
/// <summary>
/// Write out the mid-header to the stream, if any exists
/// </summary>
public override void WriteMidHeader()
{
// This call is a no-op for textfile output
}
/// <summary>
/// Write out the separator to the stream, if any exists
/// </summary>
public override void WriteMidSeparator()
{
// This call is a no-op for textfile output
sw.Write(line);
sw.Flush();
}
/// <summary>
/// Write out the footer-separator to the stream, if any exists
/// </summary>
public override void WriteFooterSeparator()
/// <param name="sw">StreamWriter to write to</param>
private void WriteFooterSeparator(StreamWriter sw)
{
_writer.Write("\n");
_writer.Flush();
}
/// <summary>
/// Write out the footer to the stream, if any exists
/// </summary>
public override void WriteFooter()
{
// This call is a no-op for textfile output
sw.Write("\n");
sw.Flush();
}
}
}

6
SabreTools/Features/Stats.cs
View File

@@ -58,11 +58,11 @@ The stats that are outputted are as follows:
filename = Path.GetFileName(filename);
}
Statistics.OutputStats(
Inputs,
var statistics = Statistics.CalculateStatistics(Inputs, GetBoolean(features, IndividualValue));
Statistics.Write(
statistics,
filename,
OutputDir,
GetBoolean(features, IndividualValue),
GetBoolean(features, BaddumpColumnValue),
GetBoolean(features, NodumpColumnValue),
GetStatReportFormat(features));