claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split DatFiles namespace

Browse Source
This commit is contained in:
Matt Nadareski
2020-12-10 23:24:09 -08:00
parent 24e73489d2
commit 24d4be0571
37 changed files with 426 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -14,6 +14,8 @@
/SabreTools.DatFiles/obj/
/SabreTools.DatItems/bin/
/SabreTools.DatItems/obj/
/SabreTools.DatTools/bin/
/SabreTools.DatTools/obj/
/SabreTools.FileTypes/bin/
/SabreTools.FileTypes/obj/
/SabreTools.Filtering/bin/

1
RombaSharp/Features/Archive.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.DatTools;
using SabreTools.Help;
using Microsoft.Data.Sqlite;

1
RombaSharp/Features/BaseFeature.cs
View File

@@ -9,6 +9,7 @@ using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.DatTools;
using SabreTools.FileTypes;
using SabreTools.Help;
using SabreTools.Logging;

1
RombaSharp/Features/Build.cs
View File

@@ -4,6 +4,7 @@ using System.Linq;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;

4
RombaSharp/Features/DatStats.cs
View File

@@ -2,7 +2,7 @@
using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
namespace RombaSharp.Features
@@ -30,7 +30,7 @@ namespace RombaSharp.Features
Inputs = new List<string> { Path.GetFullPath(_dats) };
// Now output the stats for all inputs
ItemDictionary.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile);
Statistics.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile);
}
}
}

1
RombaSharp/Features/Diffdat.cs
View File

@@ -2,6 +2,7 @@
using System.IO;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;

1
RombaSharp/Features/Dir2Dat.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Filtering;
using SabreTools.Help;
using SabreTools.IO;

1
RombaSharp/Features/EDiffdat.cs
View File

@@ -2,6 +2,7 @@
using System.IO;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;

1
RombaSharp/Features/Miss.cs
View File

@@ -3,6 +3,7 @@ using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;

1
RombaSharp/Features/RefreshDats.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.Logging;
using Microsoft.Data.Sqlite;

1
RombaSharp/Features/RescanDepots.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.DatTools;
using SabreTools.Help;
using Microsoft.Data.Sqlite;

1
RombaSharp/RombaSharp.csproj
View File

@@ -19,6 +19,7 @@
<ItemGroup>
<ProjectReference Include="..\SabreTools.DatFiles\SabreTools.DatFiles.csproj" />
<ProjectReference Include="..\SabreTools.DatItems\SabreTools.DatItems.csproj" />
<ProjectReference Include="..\SabreTools.DatTools\SabreTools.DatTools.csproj" />
<ProjectReference Include="..\SabreTools.FileTypes\SabreTools.FileTypes.csproj" />
<ProjectReference Include="..\SabreTools.Filtering\SabreTools.Filtering.csproj" />
<ProjectReference Include="..\SabreTools.Help\SabreTools.Help.csproj" />

33
SabreTools.Core/Tools/Utilities.cs
View File

@@ -146,6 +146,39 @@ namespace SabreTools.Core.Tools
return path;
}
/// <summary>
/// Get if the given path has a valid DAT extension
/// </summary>
/// <param name="path">Path to check</param>
/// <returns>True if the extension is valid, false otherwise</returns>
public static bool HasValidDatExtension(string path)
{
// Get the extension from the path, if possible
string ext = Path.GetExtension(path).TrimStart('.');
// Check against the list of known DAT extensions
switch (ext)
{
case "csv":
case "dat":
case "json":
case "md5":
case "ripemd160":
case "sfv":
case "sha1":
case "sha256":
case "sha384":
case "sha512":
case "ssv":
case "tsv":
case "txt":
case "xml":
return true;
default:
return false;
}
}
/// Indicates whether the specified array is null or has a length of zero
/// </summary>
/// <param name="array">The array to test</param>

11
SabreTools.DatFiles/DatFile.cs
View File

@@ -16,17 +16,8 @@ namespace SabreTools.DatFiles
/// <summary>
/// Represents a format-agnostic DAT
/// </summary>
/// <remarks>
/// The fact that this one class could be separated into as many partial
/// classes as it did means that the functionality here should probably
/// be split out into either separate classes or even an entirely separate
/// namespace. Also, with that in mind, each of the individual DatFile types
/// probably should only need to inherit from a thin abstract class and
/// should not be exposed as part of the library, instead being taken care
/// of behind the scenes as part of the reading and writing.
/// </remarks>
[JsonObject("datfile"), XmlRoot("datfile")]
public abstract partial class DatFile
public abstract class DatFile
{
#region Fields

3
SabreTools.DatFiles/DatHeader.cs
View File

@@ -7,7 +7,6 @@ using System.Xml.Serialization;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles.Formats;
using SabreTools.IO;
using Newtonsoft.Json;
namespace SabreTools.DatFiles
@@ -1061,7 +1060,7 @@ namespace SabreTools.DatFiles
string filename = string.IsNullOrWhiteSpace(FileName) ? Description : FileName;
// Strip off the extension if it's a holdover from the DAT
if (Parser.HasValidDatExtension(filename))
if (Utilities.HasValidDatExtension(filename))
filename = Path.GetFileNameWithoutExtension(filename);
string outfile = $"{outDir}{filename}{extension}";

380
SabreTools.DatFiles/ItemDictionary.cs
View File

@@ -1,18 +1,13 @@
using System;
using System.Collections;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using System.Xml.Serialization;
using SabreTools.Core;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
using SabreTools.DatFiles.Reports;
using NaturalSort;
using Newtonsoft.Json;
@@ -282,7 +277,7 @@ namespace SabreTools.DatFiles
/// </summary>
/// <remarks>Special count only used by statistics output</remarks>
[JsonIgnore, XmlIgnore]
public long GameCount { get; private set; } = 0;
public long GameCount { get; set; } = 0;
/// <summary>
/// Total uncompressed size
@@ -374,8 +369,6 @@ namespace SabreTools.DatFiles
#endregion
#region Instance methods
#region Accessors
/// <summary>
@@ -462,6 +455,47 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Add statistics from another DatStats object
/// </summary>
/// <param name="stats">DatStats object to add from</param>
public void AddStatistics(ItemDictionary stats)
{
TotalCount += stats.Count;
ArchiveCount += stats.ArchiveCount;
BiosSetCount += stats.BiosSetCount;
ChipCount += stats.ChipCount;
DiskCount += stats.DiskCount;
MediaCount += stats.MediaCount;
ReleaseCount += stats.ReleaseCount;
RomCount += stats.RomCount;
SampleCount += stats.SampleCount;
GameCount += stats.GameCount;
TotalSize += stats.TotalSize;
// Individual hash counts
CRCCount += stats.CRCCount;
MD5Count += stats.MD5Count;
#if NET_FRAMEWORK
RIPEMD160Count += stats.RIPEMD160Count;
#endif
SHA1Count += stats.SHA1Count;
SHA256Count += stats.SHA256Count;
SHA384Count += stats.SHA384Count;
SHA512Count += stats.SHA512Count;
SpamSumCount += stats.SpamSumCount;
// Individual status counts
BaddumpCount += stats.BaddumpCount;
GoodCount += stats.GoodCount;
NodumpCount += stats.NodumpCount;
RemovedCount += stats.RemovedCount;
VerifiedCount += stats.VerifiedCount;
}
/// <summary>
/// Get if the file dictionary contains the key
/// </summary>
@@ -746,47 +780,6 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Add statistics from another DatStats object
/// </summary>
/// <param name="stats">DatStats object to add from</param>
private void AddStatistics(ItemDictionary stats)
{
TotalCount += stats.Count;
ArchiveCount += stats.ArchiveCount;
BiosSetCount += stats.BiosSetCount;
ChipCount += stats.ChipCount;
DiskCount += stats.DiskCount;
MediaCount += stats.MediaCount;
ReleaseCount += stats.ReleaseCount;
RomCount += stats.RomCount;
SampleCount += stats.SampleCount;
GameCount += stats.GameCount;
TotalSize += stats.TotalSize;
// Individual hash counts
CRCCount += stats.CRCCount;
MD5Count += stats.MD5Count;
#if NET_FRAMEWORK
RIPEMD160Count += stats.RIPEMD160Count;
#endif
SHA1Count += stats.SHA1Count;
SHA256Count += stats.SHA256Count;
SHA384Count += stats.SHA384Count;
SHA512Count += stats.SHA512Count;
SpamSumCount += stats.SpamSumCount;
// Individual status counts
BaddumpCount += stats.BaddumpCount;
GoodCount += stats.GoodCount;
NodumpCount += stats.NodumpCount;
RemovedCount += stats.RemovedCount;
VerifiedCount += stats.VerifiedCount;
}
/// <summary>
/// Ensure the key exists in the items dictionary
/// </summary>
@@ -1194,6 +1187,44 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Reset all statistics
/// </summary>
public void ResetStatistics()
{
TotalCount = 0;
ArchiveCount = 0;
BiosSetCount = 0;
ChipCount = 0;
DiskCount = 0;
MediaCount = 0;
ReleaseCount = 0;
RomCount = 0;
SampleCount = 0;
GameCount = 0;
TotalSize = 0;
CRCCount = 0;
MD5Count = 0;
#if NET_FRAMEWORK
RIPEMD160Count = 0;
#endif
SHA1Count = 0;
SHA256Count = 0;
SHA384Count = 0;
SHA512Count = 0;
SpamSumCount = 0;
BaddumpCount = 0;
GoodCount = 0;
NodumpCount = 0;
RemovedCount = 0;
VerifiedCount = 0;
}
/// <summary>
/// Get the highest-order Field value that represents the statistics
/// </summary>
@@ -1230,44 +1261,6 @@ namespace SabreTools.DatFiles
return Field.DatItem_CRC;
}
/// <summary>
/// Reset all statistics
/// </summary>
private void ResetStatistics()
{
TotalCount = 0;
ArchiveCount = 0;
BiosSetCount = 0;
ChipCount = 0;
DiskCount = 0;
MediaCount = 0;
ReleaseCount = 0;
RomCount = 0;
SampleCount = 0;
GameCount = 0;
TotalSize = 0;
CRCCount = 0;
MD5Count = 0;
#if NET_FRAMEWORK
RIPEMD160Count = 0;
#endif
SHA1Count = 0;
SHA256Count = 0;
SHA384Count = 0;
SHA512Count = 0;
SpamSumCount = 0;
BaddumpCount = 0;
GoodCount = 0;
NodumpCount = 0;
RemovedCount = 0;
VerifiedCount = 0;
}
/// <summary>
/// Sort the input DAT and get the key to be used by the item
/// </summary>
@@ -1335,216 +1328,5 @@ namespace SabreTools.DatFiles
}
#endregion
#endregion // Instance methods
#region Static methods
#region Writing
/// <summary>
/// Output the stats for a list of input dats as files in a human-readable format
/// </summary>
/// <param name="inputs">List of input files and folders</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="single">True if single DAT stats are output, false otherwise</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat" > Set the statistics output format to use</param>
public static void OutputStats(
List<string> inputs,
string reportName,
string outDir,
bool single,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
statDatFormat = StatReportFormat.Textfile;
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Get the proper output directory name
outDir = outDir.Ensure();
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outputs = CreateOutStatsNames(outDir, statDatFormat, reportName);
// Make sure we have all files and then order them
List<ParentablePath> files = PathTool.GetFilesOnly(inputs);
files = files
.OrderBy(i => Path.GetDirectoryName(i.CurrentPath))
.ThenBy(i => Path.GetFileName(i.CurrentPath))
.ToList();
// Get all of the writers that we need
List<BaseReport> reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList();
// Write the header, if any
reports.ForEach(report => report.WriteHeader());
// Init all total variables
ItemDictionary totalStats = new ItemDictionary();
// Init directory-level variables
string lastdir = null;
string basepath = null;
ItemDictionary dirStats = new ItemDictionary();
// Now process each of the input files
foreach (ParentablePath file in files)
{
// Get the directory for the current file
string thisdir = Path.GetDirectoryName(file.CurrentPath);
basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath));
// If we don't have the first file and the directory has changed, show the previous directory stats and reset
if (lastdir != null && thisdir != lastdir)
{
// Output separator if needed
reports.ForEach(report => report.WriteMidSeparator());
DatFile lastdirdat = DatFile.Create();
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
}
staticLogger.Verbose($"Beginning stat collection for '{file.CurrentPath}'");
List<string> games = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath);
datdata.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
staticLogger.User($"Adding stats for file '{file.CurrentPath}'\n");
if (single)
{
reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items));
reports.ForEach(report => report.Write());
}
// Add single DAT stats to dir
dirStats.AddStatistics(datdata.Items);
dirStats.GameCount += datdata.Items.Keys.Count();
// Add single DAT stats to totals
totalStats.AddStatistics(datdata.Items);
totalStats.GameCount += datdata.Items.Keys.Count();
// Make sure to assign the new directory
lastdir = thisdir;
}
// Output the directory stats one last time
reports.ForEach(report => report.WriteMidSeparator());
if (single)
{
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
}
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
// Output total DAT stats
reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats));
reports.ForEach(report => report.Write());
// Output footer if needed
reports.ForEach(report => report.WriteFooter());
staticLogger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen");
}
/// <summary>
/// Get the proper extension for the stat output format
/// </summary>
/// <param name="outDir">Output path to use</param>
/// <param name="statDatFormat">StatDatFormat to get the extension for</param>
/// <param name="reportName">Name of the input file to use</param>
/// <returns>Dictionary of output formats mapped to file names</returns>
private static Dictionary<StatReportFormat, string> CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true)
{
Dictionary<StatReportFormat, string> output = new Dictionary<StatReportFormat, string>();
// First try to create the output directory if we need to
if (!Directory.Exists(outDir))
Directory.CreateDirectory(outDir);
// Double check the outDir for the end delim
if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString()))
outDir += Path.DirectorySeparatorChar;
// For each output format, get the appropriate stream writer
output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.Textfile))
output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.CSV))
output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.HTML))
output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.SSV))
output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.TSV))
output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite));
return output;
}
/// <summary>
/// Help generating the outstats name
/// </summary>
/// <param name="outDir">Output directory</param>
/// <param name="extension">Extension to use for the file</param>
/// <param name="reportName">Name of the input file to use</param>
/// <param name="overwrite">True if we ignore existing files, false otherwise</param>
/// <returns>String containing the new filename</returns>
private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite)
{
string outfile = outDir + reportName + extension;
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
if (!overwrite)
{
int i = 1;
while (File.Exists(outfile))
{
outfile = $"{outDir}{reportName}_{i}{extension}";
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
i++;
}
}
return outfile;
}
#endregion
#endregion // Static methods
}
}

2
SabreTools.DatFiles/Reports/BaseReport.cs
View File

@@ -2,8 +2,8 @@
using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
// TODO: Reports namespace no longer is circular with DatFiles
namespace SabreTools.DatFiles.Reports
{
/// <summary>

4
SabreTools.DatFiles/SabreTools.DatFiles.csproj
View File

@@ -14,16 +14,12 @@
<ItemGroup>
<ProjectReference Include="..\SabreTools.Core\SabreTools.Core.csproj" />
<ProjectReference Include="..\SabreTools.DatItems\SabreTools.DatItems.csproj" />
<ProjectReference Include="..\SabreTools.FileTypes\SabreTools.FileTypes.csproj" />
<ProjectReference Include="..\SabreTools.Filtering\SabreTools.Filtering.csproj" />
<ProjectReference Include="..\SabreTools.IO\SabreTools.IO.csproj" />
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />
<ProjectReference Include="..\SabreTools.Skippers\SabreTools.Skippers.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" />
</ItemGroup>
</Project>

3
SabreTools.DatFiles/DatFromDir.cs → SabreTools.DatTools/DatFromDir.cs
View File

@@ -5,13 +5,14 @@ using System.Threading;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.IO;
using SabreTools.Logging;
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
/// <summary>
/// This file represents all methods related to populating a DatFile

4
SabreTools.DatFiles/DatTool.cs → SabreTools.DatTools/DatTool.cs
View File

@@ -4,12 +4,12 @@ using System.Linq;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
// TODO: Should each of the individual pieces of partial classes be their own classes?
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
// This file represents all methods related to converting and updating DatFiles
public class DatTool

3
SabreTools.DatFiles/Modification.cs → SabreTools.DatTools/Modification.cs
View File

@@ -7,13 +7,14 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.Filtering;
using SabreTools.IO;
using SabreTools.Logging;
// This file represents all methods related to the Filtering namespace
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
public class Modification
{

41
SabreTools.DatFiles/Parser.cs → SabreTools.DatTools/Parser.cs
View File

@@ -3,11 +3,13 @@ using System.IO;
using System.Text.RegularExpressions;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.IO;
using SabreTools.Logging;
// This file represents all methods related to parsing from a file
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
public class Parser
{
@@ -78,7 +80,7 @@ namespace SabreTools.DatFiles
string currentPath = input.CurrentPath;
// Check the file extension first as a safeguard
if (!HasValidDatExtension(currentPath))
if (!Utilities.HasValidDatExtension(currentPath))
return;
// If the output filename isn't set already, get the internal filename
@@ -105,39 +107,6 @@ namespace SabreTools.DatFiles
}
}
/// <summary>
/// Get if the given path has a valid DAT extension
/// </summary>
/// <param name="path">Path to check</param>
/// <returns>True if the extension is valid, false otherwise</returns>
public static bool HasValidDatExtension(string path)
{
// Get the extension from the path, if possible
string ext = path.GetNormalizedExtension();
// Check against the list of known DAT extensions
switch (ext)
{
case "csv":
case "dat":
case "json":
case "md5":
case "ripemd160":
case "sfv":
case "sha1":
case "sha256":
case "sha384":
case "sha512":
case "ssv":
case "tsv":
case "txt":
case "xml":
return true;
default:
return false;
}
}
/// <summary>
/// Get what type of DAT the input file is
/// </summary>
@@ -146,7 +115,7 @@ namespace SabreTools.DatFiles
private static DatFormat GetDatFormat(string filename)
{
// Limit the output formats based on extension
if (!HasValidDatExtension(filename))
if (!Utilities.HasValidDatExtension(filename))
return 0;
// Get the extension from the filename

3
SabreTools.DatFiles/Rebuilder.cs → SabreTools.DatTools/Rebuilder.cs
View File

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
@@ -13,7 +14,7 @@ using SabreTools.Logging;
using SabreTools.Skippers;
// This file represents all methods related to rebuilding from a DatFile
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
public class Rebuilder
{

25
SabreTools.DatTools/SabreTools.DatTools.csproj Normal file
View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net48;netcoreapp3.1;net5.0</TargetFrameworks>
<RuntimeIdentifiers>win10-x64;win7-x86</RuntimeIdentifiers>
<Configurations>Debug;Release</Configurations>
<Platforms>AnyCPU;x64</Platforms>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net48'">
<DefineConstants>NET_FRAMEWORK</DefineConstants>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\SabreTools.Core\SabreTools.Core.csproj" />
<ProjectReference Include="..\SabreTools.DatFiles\SabreTools.DatFiles.csproj" />
<ProjectReference Include="..\SabreTools.DatItems\SabreTools.DatItems.csproj" />
<ProjectReference Include="..\SabreTools.FileTypes\SabreTools.FileTypes.csproj" />
<ProjectReference Include="..\SabreTools.Filtering\SabreTools.Filtering.csproj" />
<ProjectReference Include="..\SabreTools.IO\SabreTools.IO.csproj" />
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />
<ProjectReference Include="..\SabreTools.Skippers\SabreTools.Skippers.csproj" />
</ItemGroup>
</Project>

3
SabreTools.DatFiles/Splitter.cs → SabreTools.DatTools/Splitter.cs
View File

@@ -6,13 +6,14 @@ using System.Net;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.IO;
using SabreTools.Logging;
using NaturalSort;
// This file represents all methods related to splitting a DatFile into multiple
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
// TODO: Implement Level split
public class Splitter

227
SabreTools.DatTools/Statistics.cs Normal file
View File

@@ -0,0 +1,227 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatFiles.Reports;
using SabreTools.IO;
using SabreTools.Logging;
namespace SabreTools.DatTools
{
public class Statistics
{
#region Logging
/// <summary>
/// Logging object
/// </summary>
private static readonly Logger logger = new Logger();
#endregion
/// <summary>
/// Output the stats for a list of input dats as files in a human-readable format
/// </summary>
/// <param name="inputs">List of input files and folders</param>
/// <param name="reportName">Name of the output file</param>
/// <param name="single">True if single DAT stats are output, false otherwise</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat" > Set the statistics output format to use</param>
public static void OutputStats(
List<string> inputs,
string reportName,
string outDir,
bool single,
bool baddumpCol,
bool nodumpCol,
StatReportFormat statDatFormat)
{
// If there's no output format, set the default
if (statDatFormat == StatReportFormat.None)
statDatFormat = StatReportFormat.Textfile;
// Get the proper output file name
if (string.IsNullOrWhiteSpace(reportName))
reportName = "report";
// Get the proper output directory name
outDir = outDir.Ensure();
// Get the dictionary of desired output report names
Dictionary<StatReportFormat, string> outputs = CreateOutStatsNames(outDir, statDatFormat, reportName);
// Make sure we have all files and then order them
List<ParentablePath> files = PathTool.GetFilesOnly(inputs);
files = files
.OrderBy(i => Path.GetDirectoryName(i.CurrentPath))
.ThenBy(i => Path.GetFileName(i.CurrentPath))
.ToList();
// Get all of the writers that we need
List<BaseReport> reports = outputs.Select(kvp => BaseReport.Create(kvp.Key, kvp.Value, baddumpCol, nodumpCol)).ToList();
// Write the header, if any
reports.ForEach(report => report.WriteHeader());
// Init all total variables
ItemDictionary totalStats = new ItemDictionary();
// Init directory-level variables
string lastdir = null;
string basepath = null;
ItemDictionary dirStats = new ItemDictionary();
// Now process each of the input files
foreach (ParentablePath file in files)
{
// Get the directory for the current file
string thisdir = Path.GetDirectoryName(file.CurrentPath);
basepath = Path.GetDirectoryName(Path.GetDirectoryName(file.CurrentPath));
// If we don't have the first file and the directory has changed, show the previous directory stats and reset
if (lastdir != null && thisdir != lastdir)
{
// Output separator if needed
reports.ForEach(report => report.WriteMidSeparator());
DatFile lastdirdat = DatFile.Create();
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
}
logger.Verbose($"Beginning stat collection for '{file.CurrentPath}'");
List<string> games = new List<string>();
DatFile datdata = Parser.CreateAndParse(file.CurrentPath);
datdata.Items.BucketBy(Field.Machine_Name, DedupeType.None, norename: true);
// Output single DAT stats (if asked)
logger.User($"Adding stats for file '{file.CurrentPath}'\n");
if (single)
{
reports.ForEach(report => report.ReplaceStatistics(datdata.Header.FileName, datdata.Items.Keys.Count, datdata.Items));
reports.ForEach(report => report.Write());
}
// Add single DAT stats to dir
dirStats.AddStatistics(datdata.Items);
dirStats.GameCount += datdata.Items.Keys.Count();
// Add single DAT stats to totals
totalStats.AddStatistics(datdata.Items);
totalStats.GameCount += datdata.Items.Keys.Count();
// Make sure to assign the new directory
lastdir = thisdir;
}
// Output the directory stats one last time
reports.ForEach(report => report.WriteMidSeparator());
if (single)
{
reports.ForEach(report => report.ReplaceStatistics($"DIR: {WebUtility.HtmlEncode(lastdir)}", dirStats.GameCount, dirStats));
reports.ForEach(report => report.Write());
}
// Write the mid-footer, if any
reports.ForEach(report => report.WriteFooterSeparator());
// Write the header, if any
reports.ForEach(report => report.WriteMidHeader());
// Reset the directory stats
dirStats.ResetStatistics();
// Output total DAT stats
reports.ForEach(report => report.ReplaceStatistics("DIR: All DATs", totalStats.GameCount, totalStats));
reports.ForEach(report => report.Write());
// Output footer if needed
reports.ForEach(report => report.WriteFooter());
logger.User($"{Environment.NewLine}Please check the log folder if the stats scrolled offscreen");
}
/// <summary>
/// Get the proper extension for the stat output format
/// </summary>
/// <param name="outDir">Output path to use</param>
/// <param name="statDatFormat">StatDatFormat to get the extension for</param>
/// <param name="reportName">Name of the input file to use</param>
/// <returns>Dictionary of output formats mapped to file names</returns>
private static Dictionary<StatReportFormat, string> CreateOutStatsNames(string outDir, StatReportFormat statDatFormat, string reportName, bool overwrite = true)
{
Dictionary<StatReportFormat, string> output = new Dictionary<StatReportFormat, string>();
// First try to create the output directory if we need to
if (!Directory.Exists(outDir))
Directory.CreateDirectory(outDir);
// Double check the outDir for the end delim
if (!outDir.EndsWith(Path.DirectorySeparatorChar.ToString()))
outDir += Path.DirectorySeparatorChar;
// For each output format, get the appropriate stream writer
output.Add(StatReportFormat.None, CreateOutStatsNamesHelper(outDir, ".null", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.Textfile))
output.Add(StatReportFormat.Textfile, CreateOutStatsNamesHelper(outDir, ".txt", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.CSV))
output.Add(StatReportFormat.CSV, CreateOutStatsNamesHelper(outDir, ".csv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.HTML))
output.Add(StatReportFormat.HTML, CreateOutStatsNamesHelper(outDir, ".html", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.SSV))
output.Add(StatReportFormat.SSV, CreateOutStatsNamesHelper(outDir, ".ssv", reportName, overwrite));
if (statDatFormat.HasFlag(StatReportFormat.TSV))
output.Add(StatReportFormat.TSV, CreateOutStatsNamesHelper(outDir, ".tsv", reportName, overwrite));
return output;
}
/// <summary>
/// Help generating the outstats name
/// </summary>
/// <param name="outDir">Output directory</param>
/// <param name="extension">Extension to use for the file</param>
/// <param name="reportName">Name of the input file to use</param>
/// <param name="overwrite">True if we ignore existing files, false otherwise</param>
/// <returns>String containing the new filename</returns>
private static string CreateOutStatsNamesHelper(string outDir, string extension, string reportName, bool overwrite)
{
string outfile = outDir + reportName + extension;
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
if (!overwrite)
{
int i = 1;
while (File.Exists(outfile))
{
outfile = $"{outDir}{reportName}_{i}{extension}";
outfile = outfile.Replace($"{Path.DirectorySeparatorChar}{Path.DirectorySeparatorChar}", Path.DirectorySeparatorChar.ToString());
i++;
}
}
return outfile;
}
}
}

3
SabreTools.DatFiles/Verification.cs → SabreTools.DatTools/Verification.cs
View File

@@ -4,13 +4,14 @@ using System.Linq;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.Logging;
// This file represents all methods related to verifying with a DatFile
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
public class Verification
{

3
SabreTools.DatFiles/Writer.cs → SabreTools.DatTools/Writer.cs
View File

@@ -4,12 +4,13 @@ using System.Linq;
using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatFiles.Reports;
using SabreTools.IO;
using SabreTools.Logging;
// This file represents all methods related to writing to a file
namespace SabreTools.DatFiles
namespace SabreTools.DatTools
{
public class Writer
{

10
SabreTools.sln
View File

@@ -32,6 +32,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.FileTypes", "Sab
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.DatItems", "SabreTools.DatItems\SabreTools.DatItems.csproj", "{90ADE461-33B1-4E0D-925F-C99913665F0C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SabreTools.DatTools", "SabreTools.DatTools\SabreTools.DatTools.csproj", "{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -128,6 +130,14 @@ Global
{90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|Any CPU.Build.0 = Release|Any CPU
{90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|x64.ActiveCfg = Release|Any CPU
{90ADE461-33B1-4E0D-925F-C99913665F0C}.Release|x64.Build.0 = Release|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|x64.ActiveCfg = Debug|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Debug|x64.Build.0 = Debug|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|Any CPU.Build.0 = Release|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|x64.ActiveCfg = Release|Any CPU
{E0D12252-BBF3-4E3C-B2E2-79FA49EE31E5}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

3
SabreTools/Features/Batch.cs
View File

@@ -7,6 +7,7 @@ using System.Text.RegularExpressions;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Filtering;
using SabreTools.Help;
using SabreTools.IO;
@@ -153,7 +154,7 @@ Reset the internal state: reset();";
// Assume there could be multiple
foreach (string input in command.Arguments)
{
DatFiles.DatFromDir.PopulateFromDir(datFile, input);
DatTools.DatFromDir.PopulateFromDir(datFile, input);
}
// TODO: We might not want to remove higher order hashes in the future

4
SabreTools/Features/DatFromDir.cs
View File

@@ -4,7 +4,9 @@ using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
namespace SabreTools.Features
{
internal class DatFromDir : BaseFeature
@@ -89,7 +91,7 @@ namespace SabreTools.Features
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = DatFiles.DatFromDir.PopulateFromDir(
bool success = DatTools.DatFromDir.PopulateFromDir(
datdata,
basePath,
asFiles,

1
SabreTools/Features/Sort.cs
View File

@@ -3,6 +3,7 @@ using System.IO;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;
using SabreTools.Logging;

1
SabreTools/Features/Split.cs
View File

@@ -3,6 +3,7 @@ using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;
using SabreTools.Logging;

4
SabreTools/Features/Stats.cs
View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
namespace SabreTools.Features
@@ -55,7 +55,7 @@ The stats that are outputted are as follows:
filename = Path.GetFileName(filename);
}
ItemDictionary.OutputStats(
Statistics.OutputStats(
Inputs,
filename,
OutputDir,

1
SabreTools/Features/Update.cs
View File

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;
using SabreTools.Logging;

5
SabreTools/Features/Verify.cs
View File

@@ -2,6 +2,7 @@
using SabreTools.Core;
using SabreTools.DatFiles;
using SabreTools.DatTools;
using SabreTools.Help;
using SabreTools.IO;
using SabreTools.Logging;
@@ -84,7 +85,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatFiles.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
Verification.VerifyGeneric(datdata, hashOnly);
@@ -133,7 +134,7 @@ namespace SabreTools.Features
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
DatFiles.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
DatTools.DatFromDir.PopulateFromDir(datdata, input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
Verification.VerifyGeneric(datdata, hashOnly);

1
SabreTools/SabreTools.csproj
View File

@@ -18,6 +18,7 @@
<ItemGroup>
<ProjectReference Include="..\SabreTools.DatFiles\SabreTools.DatFiles.csproj" />
<ProjectReference Include="..\SabreTools.DatTools\SabreTools.DatTools.csproj" />
<ProjectReference Include="..\SabreTools.Filtering\SabreTools.Filtering.csproj" />
<ProjectReference Include="..\SabreTools.Help\SabreTools.Help.csproj" />
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />