claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

RombaSharp split Features too

Browse Source
This commit is contained in:
Matt Nadareski
2020-08-01 13:25:32 -07:00
parent 4bd0c10b0a
commit b321c38be9
29 changed files with 2158 additions and 1915 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
RombaSharp/Features/Archive.cs Normal file
View File

@@ -0,0 +1,198 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Help;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class Archive : BaseFeature
{
public const string Value = "Archive";
public Archive()
{
Name = Value;
Flags = new List<string>() { "archive" };
Description = "Adds ROM files from the specified directories to the ROM archive.";
_featureType = FeatureType.Flag;
LongDescription = @"Adds ROM files from the specified directories to the ROM archive.
Traverses the specified directory trees looking for zip files and normal files.
Unpacked files will be stored as individual entries. Prior to unpacking a zip
file, the external SHA1 is checked against the DAT index.
If -only-needed is set, only those files are put in the ROM archive that
have a current entry in the DAT index.";
Features = new Dictionary<string, Feature>();
AddFeature(OnlyNeededFlag);
AddFeature(ResumeStringInput);
AddFeature(IncludeZipsInt32Input); // Defaults to 0
AddFeature(WorkersInt32Input);
AddFeature(IncludeGZipsInt32Input); // Defaults to 0
AddFeature(Include7ZipsInt32Input); // Defaults to 0
AddFeature(SkipInitialScanFlag);
AddFeature(UseGolangZipFlag);
AddFeature(NoDbFlag);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get the archive scanning level
// TODO: Remove usage
int sevenzip = GetInt32(features, Include7ZipsInt32Value);
int gz = GetInt32(features, IncludeGZipsInt32Value);
int zip = GetInt32(features, IncludeZipsInt32Value);
// Get feature flags
bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
// First we want to get just all directories from the inputs
List<string> onlyDirs = new List<string>();
foreach (string input in Inputs)
{
if (Directory.Exists(input))
onlyDirs.Add(Path.GetFullPath(input));
}
// Then process all of the input directories into an internal DAT
DatFile df = DatFile.Create();
foreach (string dir in onlyDirs)
{
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
df.PopulateFromDir(dir, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null);
df.PopulateFromDir(dir, Hash.DeepHashes, false, true, SkipFileType.None, false, false, _tmpdir, false, null, true, null);
}
// Create an empty Dat for files that need to be rebuilt
DatFile need = DatFile.Create();
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the Dats, add the files to the database
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
foreach (string key in df.Items.Keys)
{
List<DatItem> datItems = df.Items[key];
foreach (Rom rom in datItems)
{
// If we care about if the file exists, check the databse first
if (onlyNeeded && !noDb)
{
string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
+ $" WHERE crcsha1.crc=\"{rom.CRC}\""
+ $" OR md5sha1.md5=\"{rom.MD5}\""
+ $" OR md5sha1.sha1=\"{rom.SHA1}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
// Add to the queries
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcquery += $" (\"{rom.CRC}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5query += $" (\"{rom.MD5}\"),";
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
// Otherwise, just add the file to the list
else
{
// Add to the queries
if (!noDb)
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcquery += $" (\"{rom.CRC}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5query += $" (\"{rom.MD5}\"),";
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
}
// Now run the queries, if they're populated
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
// Create the sorting object to use and rebuild the needed files
need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], false /*quickScan*/, false /*date*/,
false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzipRomba, false /*updateDat*/,
null /*headerToCheckAgainst*/, true /* chdsAsFiles */);
}
}
}

370
RombaSharp/RombaSharp.Helpers.cs → RombaSharp/Features/BaseFeature.cs
View File

@@ -7,13 +7,375 @@ using System.Xml;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
using Microsoft.Data.Sqlite;
namespace RombaSharp
namespace RombaSharp.Features
{
public partial class RombaSharp
internal class BaseFeature : TopLevel
{
#region Private Flag features
internal const string CopyValue = "copy";
internal static Feature CopyFlag
{
get
{
return new Feature(
CopyValue,
"-copy",
"Copy files to output instead of rebuilding",
FeatureType.Flag);
}
} // Unique to RombaSharp
internal const string FixdatOnlyValue = "fixdat-only";
internal static Feature FixdatOnlyFlag
{
get
{
return new Feature(
FixdatOnlyValue,
"-fixdatOnly",
"only fix dats and don't generate torrentzips",
FeatureType.Flag);
}
}
internal const string LogOnlyValue = "log-only";
internal static Feature LogOnlyFlag
{
get
{
return new Feature(
LogOnlyValue,
"-log-only",
"Only write out actions to log",
FeatureType.Flag);
}
}
internal const string NoDbValue = "no-db";
internal static Feature NoDbFlag
{
get
{
return new Feature(
NoDbValue,
"-no-db",
"archive into depot but do not touch DB index and ignore only-needed flag",
FeatureType.Flag);
}
}
internal const string OnlyNeededValue = "only-needed";
internal static Feature OnlyNeededFlag
{
get
{
return new Feature(
OnlyNeededValue,
"-only-needed",
"only archive ROM files actually referenced by DAT files from the DAT index",
FeatureType.Flag);
}
}
internal const string SkipInitialScanValue = "skip-initial-scan";
internal static Feature SkipInitialScanFlag
{
get
{
return new Feature(
SkipInitialScanValue,
"-skip-initial-scan",
"skip the initial scan of the files to determine amount of work",
FeatureType.Flag);
}
}
internal const string UseGolangZipValue = "use-golang-zip";
internal static Feature UseGolangZipFlag
{
get
{
return new Feature(
UseGolangZipValue,
"-use-golang-zip",
"use go zip implementation instead of zlib",
FeatureType.Flag);
}
}
#endregion
#region Private Int32 features
internal const string Include7ZipsInt32Value = "include-7zips";
internal static Feature Include7ZipsInt32Input
{
get
{
return new Feature(
Include7ZipsInt32Value,
"-include-7zips",
"flag value == 0 means: add 7zip files themselves into the depot in addition to their contents, flag value == 2 means add 7zip files themselves but don't add content",
FeatureType.Int32);
}
}
internal const string IncludeGZipsInt32Value = "include-gzips";
internal static Feature IncludeGZipsInt32Input
{
get
{
return new Feature(
IncludeGZipsInt32Value,
"-include-gzips",
"flag value == 0 means: add gzip files themselves into the depot in addition to their contents, flag value == 2 means add gzip files themselves but don't add content",
FeatureType.Int32);
}
}
internal const string IncludeZipsInt32Value = "include-zips";
internal static Feature IncludeZipsInt32Input
{
get
{
return new Feature(
IncludeZipsInt32Value,
"-include-zips",
"flag value == 0 means: add zip files themselves into the depot in addition to their contents, flag value == 2 means add zip files themselves but don't add content",
FeatureType.Int32);
}
}
internal const string SubworkersInt32Value = "subworkers";
internal static Feature SubworkersInt32Input
{
get
{
return new Feature(
SubworkersInt32Value,
"-subworkers",
"how many subworkers to launch for each worker",
FeatureType.Int32);
}
} // Defaults to Workers count in config
internal const string WorkersInt32Value = "workers";
internal static Feature WorkersInt32Input
{
get
{
return new Feature(
WorkersInt32Value,
"-workers",
"how many workers to launch for the job",
FeatureType.Int32);
}
} // Defaults to Workers count in config
#endregion
#region Private Int64 features
internal const string SizeInt64Value = "size";
internal static Feature SizeInt64Input
{
get
{
return new Feature(
SizeInt64Value,
"-size",
"size of the rom to lookup",
FeatureType.Int64);
}
}
#endregion
#region Private List<String> features
internal const string DatsListStringValue = "dats";
internal static Feature DatsListStringInput
{
get
{
return new Feature(
DatsListStringValue,
"-dats",
"purge only roms declared in these dats",
FeatureType.List);
}
}
internal const string DepotListStringValue = "depot";
internal static Feature DepotListStringInput
{
get
{
return new Feature(
DepotListStringValue,
"-depot",
"work only on specified depot path",
FeatureType.List);
}
}
#endregion
#region Private String features
internal const string BackupStringValue = "backup";
internal static Feature BackupStringInput
{
get
{
return new Feature(
BackupStringValue,
"-backup",
"backup directory where backup files are moved to",
FeatureType.String);
}
}
internal const string DescriptionStringValue = "description";
internal static Feature DescriptionStringInput
{
get
{
return new Feature(
DescriptionStringValue,
"-description",
"description value in DAT header",
FeatureType.String);
}
}
internal const string MissingSha1sStringValue = "missing-sha1s";
internal static Feature MissingSha1sStringInput
{
get
{
return new Feature(
MissingSha1sStringValue,
"-missingSha1s",
"write paths of dats with missing sha1s into this file",
FeatureType.String);
}
}
internal const string NameStringValue = "name";
internal static Feature NameStringInput
{
get
{
return new Feature(
NameStringValue,
"-name",
"name value in DAT header",
FeatureType.String);
}
}
internal const string NewStringValue = "new";
internal static Feature NewStringInput
{
get
{
return new Feature(
NewStringValue,
"-new",
"new DAT file",
FeatureType.String);
}
}
internal const string OldStringValue = "old";
internal static Feature OldStringInput
{
get
{
return new Feature(
OldStringValue,
"-old",
"old DAT file",
FeatureType.String);
}
}
internal const string OutStringValue = "out";
internal static Feature OutStringInput
{
get
{
return new Feature(
OutStringValue,
"-out",
"output file",
FeatureType.String);
}
}
internal const string ResumeStringValue = "resume";
internal static Feature ResumeStringInput
{
get
{
return new Feature(
ResumeStringValue,
"-resume",
"resume a previously interrupted operation from the specified path",
FeatureType.String);
}
}
internal const string SourceStringValue = "source";
internal static Feature SourceStringInput
{
get
{
return new Feature(
SourceStringValue,
"-source",
"source directory",
FeatureType.String);
}
}
#endregion
// General settings
internal static string _logdir; // Log folder location
internal static string _tmpdir; // Temp folder location
internal static string _webdir; // Web frontend location
internal static string _baddir; // Fail-to-unpack file folder location
internal static int _verbosity; // Verbosity of the output
internal static int _cores; // Forced CPU cores
// DatRoot settings
internal static string _dats; // DatRoot folder location
internal static string _db; // Database name
// Depot settings
internal static Dictionary<string, Tuple<long, bool>> _depots; // Folder location, Max size
// Server settings
internal static int _port; // Web server port
// Other internal variables
internal const string _config = "config.xml";
internal const string _dbSchema = "rombasharp";
internal static string _connectionString;
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
InitializeConfiguration();
DatabaseTools.EnsureDatabase(_dbSchema, _db, _connectionString);
}
#region Helper methods
/// <summary>
@@ -21,7 +383,7 @@ namespace RombaSharp
/// </summary>
/// <param name="inputs">List of input strings to check for, presumably file names</param>
/// <returns>Dictionary of hash/full path for each of the valid DATs</returns>
private static Dictionary<string, string> GetValidDats(List<string> inputs)
internal static Dictionary<string, string> GetValidDats(List<string> inputs)
{
// Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not
List<string> datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList();
@@ -242,7 +604,7 @@ namespace RombaSharp
/// </summary>
/// <param name="dat">DatFile hash information to add</param>
/// <param name="dbc">Database connection to use</param>
private static void AddDatToDatabase(Rom dat, SqliteConnection dbc)
internal static void AddDatToDatabase(Rom dat, SqliteConnection dbc)
{
// Get the dat full path
string fullpath = Path.Combine(_dats, (dat.MachineName == "dats" ? string.Empty : dat.MachineName), dat.Name);

69
RombaSharp/Features/Build.cs Normal file
View File

@@ -0,0 +1,69 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class Build : BaseFeature
{
public const string Value = "Build";
public Build()
{
Name = Value;
Flags = new List<string>() { "build" };
Description = "For each specified DAT file it creates the torrentzip files.";
_featureType = FeatureType.Flag;
LongDescription = @"For each specified DAT file it creates the torrentzip files in the specified
output dir. The files will be placed in the specified location using a folder
structure according to the original DAT master directory tree structure.";
Features = new Dictionary<string, Feature>();
AddFeature(OutStringInput);
AddFeature(FixdatOnlyFlag);
AddFeature(CopyFlag);
AddFeature(WorkersInt32Input);
AddFeature(SubworkersInt32Input);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool copy = GetBoolean(features, CopyValue);
string outdat = GetString(features, OutStringValue);
// Verify the filenames
Dictionary<string, string> foundDats = GetValidDats(Inputs);
// Ensure the output directory is set
if (string.IsNullOrWhiteSpace(outdat))
outdat = "out";
// Now that we have the dictionary, we can loop through and output to a new folder for each
foreach (string key in foundDats.Keys)
{
// Get the DAT file associated with the key
DatFile datFile = DatFile.CreateAndParse(Path.Combine(_dats, foundDats[key]));
// Create the new output directory if it doesn't exist
string outputFolder = Path.Combine(outdat, Path.GetFileNameWithoutExtension(foundDats[key]));
DirectoryExtensions.Ensure(outputFolder, create: true);
// Get all online depots
List<string> onlineDepots = _depots.Where(d => d.Value.Item2).Select(d => d.Key).ToList();
// Now scan all of those depots and rebuild
datFile.RebuildDepot(onlineDepots, outputFolder, false /*date*/,
false /*delete*/, false /*inverse*/, (copy ? OutputFormat.TorrentGzipRomba : OutputFormat.TorrentZip),
false /*updateDat*/, null /*headerToCheckAgainst*/);
}
}
}
}

28
RombaSharp/Features/Cancel.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Cancel : BaseFeature
{
public const string Value = "Cancel";
public Cancel()
{
Name = Value;
Flags = new List<string>() { "cancel" };
Description = "Cancels current long-running job";
_featureType = FeatureType.Flag;
LongDescription = "Cancels current long-running job.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.User("This feature is not yet implemented: cancel");
}
}
}

36
RombaSharp/Features/DatStats.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class DatStats : BaseFeature
{
public const string Value = "DatStats";
public DatStats()
{
Name = Value;
Flags = new List<string>() { "datstats" };
Description = "Prints dat stats.";
_featureType = FeatureType.Flag;
LongDescription = "Print dat stats.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// If we have no inputs listed, we want to use datroot
if (Inputs == null || Inputs.Count == 0)
Inputs = new List<string> { Path.GetFullPath(_dats) };
// Now output the stats for all inputs
ItemDictionary.OutputStats(Inputs, "rombasharp-datstats", null /* outDir */, true /* single */, true /* baddumpCol */, true /* nodumpCol */, StatReportFormat.Textfile);
}
}
}

54
RombaSharp/Features/DbStats.cs Normal file
View File

@@ -0,0 +1,54 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class DbStats : BaseFeature
{
public const string Value = "DbStats";
public DbStats()
{
Name = Value;
Flags = new List<string>() { "dbstats" };
Description = "Prints db stats.";
_featureType = FeatureType.Flag;
LongDescription = "Print db stats.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Total number of CRCs
string query = "SELECT COUNT(*) FROM crc";
SqliteCommand slc = new SqliteCommand(query, dbc);
Globals.Logger.User($"Total CRCs: {(long)slc.ExecuteScalar()}");
// Total number of MD5s
query = "SELECT COUNT(*) FROM md5";
slc = new SqliteCommand(query, dbc);
Globals.Logger.User($"Total MD5s: {(long)slc.ExecuteScalar()}");
// Total number of SHA1s
query = "SELECT COUNT(*) FROM sha1";
slc = new SqliteCommand(query, dbc);
Globals.Logger.User($"Total SHA1s: {(long)slc.ExecuteScalar()}");
// Total number of DATs
query = "SELECT COUNT(*) FROM dat";
slc = new SqliteCommand(query, dbc);
Globals.Logger.User($"Total DATs: {(long)slc.ExecuteScalar()}");
slc.Dispose();
dbc.Dispose();
}
}
}

74
RombaSharp/Features/Diffdat.cs Normal file
View File

@@ -0,0 +1,74 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Filtering;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class Diffdat : BaseFeature
{
public const string Value = "Diffdat";
public Diffdat()
{
Name = Value;
Flags = new List<string>() { "diffdat" };
Description = "Creates a DAT file with those entries that are in -new DAT.";
_featureType = FeatureType.Flag;
LongDescription = @"Creates a DAT file with those entries that are in -new DAT file and not
in -old DAT file. Ignores those entries in -old that are not in -new.";
this.Features = new Dictionary<string, Feature>();
AddFeature(OutStringInput);
AddFeature(OldStringInput);
AddFeature(NewStringInput);
AddFeature(NameStringInput);
AddFeature(DescriptionStringInput);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string newdat = GetString(features, NewStringValue);
string olddat = GetString(features, OldStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
DirectoryExtensions.Ensure(outdat, create: true);
// Check that all required files exist
if (!File.Exists(olddat))
{
Globals.Logger.Error($"File '{olddat}' does not exist!");
return;
}
if (!File.Exists(newdat))
{
Globals.Logger.Error($"File '{newdat}' does not exist!");
return;
}
// Create the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = name;
datfile.Header.Description = description;
// Create the inputs
List<string> dats = new List<string> { newdat };
List<string> basedats = new List<string> { olddat };
// Now run the diff on the inputs
datfile.DetermineUpdateType(dats, basedats, outdat, UpdateMode.DiffAgainst, false /* inplace */, false /* skip */,
new Filter(), new List<Field>(), false /* onlySame */);
}
}
}

62
RombaSharp/Features/Dir2Dat.cs Normal file
View File

@@ -0,0 +1,62 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class Dir2Dat : BaseFeature
{
public const string Value = "Dir2Dat";
public Dir2Dat()
{
Name = Value;
Flags = new List<string>() { "dir2dat" };
Description = "Creates a DAT file for the specified input directory and saves it to the -out filename.";
_featureType = FeatureType.Flag;
LongDescription = "Creates a DAT file for the specified input directory and saves it to the -out filename.";
Features = new Dictionary<string, Feature>();
AddFeature(OutStringInput);
AddFeature(SourceStringInput);
AddFeature(NameStringInput); // Defaults to "untitled"
AddFeature(DescriptionStringInput);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string source = GetString(features, SourceStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
DirectoryExtensions.Ensure(outdat, create: true);
// Check that all required directories exist
if (!Directory.Exists(source))
{
Globals.Logger.Error($"File '{source}' does not exist!");
return;
}
// Create the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
datfile.Header.Description = description;
// Now run the D2D on the input and write out
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
datfile.PopulateFromDir(source, Hash.DeepHashes, true /* bare */, false /* archivesAsFiles */, SkipFileType.None, false /* addBlanks */,
false /* addDate */, _tmpdir, false /* copyFiles */, null /* headerToCheckAgainst */, true /* chdsAsFiles */, null /* filter */);
datfile.Write(outDir: outdat);
}
}
}

39
RombaSharp/Features/DisplayHelp.cs Normal file
View File

@@ -0,0 +1,39 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class DisplayHelp : BaseFeature
{
public const string Value = "Help";
public DisplayHelp()
{
Name = Value;
Flags = new List<string>() { "-?", "-h", "--help" };
Description = "Show this help";
_featureType = FeatureType.Flag;
LongDescription = "Built-in to most of the programs is a basic help text.";
Features = new Dictionary<string, Feature>();
}
public override bool ProcessArgs(string[] args, Help help)
{
// If we had something else after help
if (args.Length > 1)
{
help.OutputIndividualFeature(args[1]);
return true;
}
// Otherwise, show generic help
else
{
help.OutputGenericHelp();
return true;
}
}
}
}

39
RombaSharp/Features/DisplayHelpDetailed.cs Normal file
View File

@@ -0,0 +1,39 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class DisplayHelpDetailed : BaseFeature
{
public const string Value = "Help (Detailed)";
public DisplayHelpDetailed()
{
Name = Value;
Flags = new List<string>() { "-??", "-hd", "--help-detailed" };
Description = "Show this detailed help";
_featureType = FeatureType.Flag;
LongDescription = "Display a detailed help text to the screen.";
Features = new Dictionary<string, Feature>();
}
public override bool ProcessArgs(string[] args, Help help)
{
// If we had something else after help
if (args.Length > 1)
{
help.OutputIndividualFeature(args[1], includeLongDescription: true);
return true;
}
// Otherwise, show generic help
else
{
help.OutputAllHelp();
return true;
}
}
}
}

67
RombaSharp/Features/EDiffdat.cs Normal file
View File

@@ -0,0 +1,67 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Filtering;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class EDiffdat : BaseFeature
{
public const string Value = "EDiffdat";
public EDiffdat()
{
Name = Value;
Flags = new List<string>() { "ediffdat" };
Description = "Creates a DAT file with those entries that are in -new DAT.";
_featureType = FeatureType.Flag;
LongDescription = @"Creates a DAT file with those entries that are in -new DAT files and not in -old DAT files. Ignores those entries in -old that are not in -new.";
Features = new Dictionary<string, Feature>();
AddFeature(OutStringInput);
AddFeature(OldStringInput);
AddFeature(NewStringInput);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
string olddat = GetString(features, OldStringValue);
string outdat = GetString(features, OutStringValue);
string newdat = GetString(features, NewStringValue);
// Ensure the output directory
DirectoryExtensions.Ensure(outdat, create: true);
// Check that all required files exist
if (!File.Exists(olddat))
{
Globals.Logger.Error($"File '{olddat}' does not exist!");
return;
}
if (!File.Exists(newdat))
{
Globals.Logger.Error($"File '{newdat}' does not exist!");
return;
}
// Create the encapsulating datfile
DatFile datfile = DatFile.Create();
// Create the inputs
List<string> dats = new List<string> { newdat };
List<string> basedats = new List<string> { olddat };
// Now run the diff on the inputs
datfile.DetermineUpdateType(dats, basedats, outdat, UpdateMode.DiffAgainst, false /* inplace */, false /* skip */,
new Filter(), new List<Field>(), false /* onlySame */);
}
}
}

73
RombaSharp/Features/Export.cs Normal file
View File

@@ -0,0 +1,73 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class Export : BaseFeature
{
public const string Value = "Export";
// Unique to RombaSharp
public Export()
{
Name = Value;
Flags = new List<string>() { "export" };
Description = "Exports db to export.csv";
_featureType = FeatureType.Flag;
LongDescription = "Exports db to standardized export.csv";
Features = new Dictionary<string, Feature>();
}
// TODO: Add ability to say which depot the files are found in
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
StreamWriter sw = new StreamWriter(FileExtensions.TryCreate("export.csv"));
// First take care of all file hashes
sw.WriteLine("CRC,MD5,SHA-1"); // ,Depot
string query = "SELECT crcsha1.crc, md5sha1.md5, md5sha1.sha1 FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"; // md5sha1.sha1=sha1depot.sha1
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
string line = $"{sldr.GetString(0)},{sldr.GetString(1)},{sldr.GetString(2)}"; // + ",{sldr.GetString(3)}";
sw.WriteLine(line);
}
}
// Then take care of all DAT hashes
sw.WriteLine();
sw.WriteLine("DAT Hash");
query = "SELECT hash FROM dat";
slc = new SqliteCommand(query, dbc);
sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
sw.WriteLine(sldr.GetString(0));
}
}
sldr.Dispose();
slc.Dispose();
sw.Dispose();
dbc.Dispose();
}
}
}

41
RombaSharp/Features/Fixdat.cs Normal file
View File

@@ -0,0 +1,41 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Fixdat : BaseFeature
{
public const string Value = "Fixdat";
public Fixdat()
{
Name = Value;
Flags = new List<string>() { "fixdat" };
Description = "For each specified DAT file it creates a fix DAT.";
_featureType = FeatureType.Flag;
LongDescription = @"For each specified DAT file it creates a fix DAT with the missing entries for that DAT. If nothing is missing it doesn't create a fix DAT for that particular DAT.";
Features = new Dictionary<string, Feature>();
AddFeature(OutStringInput);
AddFeature(FixdatOnlyFlag); // Enabled by default
AddFeature(WorkersInt32Input);
AddFeature(SubworkersInt32Input);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
// Inputs
bool fixdatOnly = GetBoolean(features, FixdatOnlyValue);
int subworkers = GetInt32(features, SubworkersInt32Value);
int workers = GetInt32(features, WorkersInt32Value);
string outdat = GetString(features, OutStringValue);
Globals.Logger.Error("This feature is not yet implemented: fixdat");
}
}
}

124
RombaSharp/Features/Import.cs Normal file
View File

@@ -0,0 +1,124 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class Import : BaseFeature
{
public const string Value = "Import";
// Unique to RombaSharp
public Import()
{
Name = Value;
Flags = new List<string>() { "import" };
Description = "Import a database from a formatted CSV file";
_featureType = FeatureType.Flag;
LongDescription = "Import a database from a formatted CSV file";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.Error("This feature is not yet implemented: import");
// First ensure the inputs and database connection
Inputs = DirectoryExtensions.GetFilesOnly(Inputs).Select(p => p.CurrentPath).ToList();
SqliteConnection dbc = new SqliteConnection(_connectionString);
SqliteCommand slc = new SqliteCommand();
dbc.Open();
// Now, for each of these files, attempt to add the data found inside
foreach (string input in Inputs)
{
StreamReader sr = new StreamReader(FileExtensions.TryOpenRead(input));
// The first line should be the hash header
string line = sr.ReadLine();
if (line != "CRC,MD5,SHA-1") // ,Depot
{
Globals.Logger.Error("{0} is not a valid export file");
continue;
}
// Define the insert queries
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// For each line until we hit a blank line...
while (!sr.EndOfStream && line != string.Empty)
{
line = sr.ReadLine();
string[] hashes = line.Split(',');
// Loop through the parsed entries
if (!string.IsNullOrWhiteSpace(hashes[0]))
crcquery += $" (\"{hashes[0]}\"),";
if (!string.IsNullOrWhiteSpace(hashes[1]))
md5query += $" (\"{hashes[1]}\"),";
if (!string.IsNullOrWhiteSpace(hashes[2]))
{
sha1query += $" (\"{hashes[2]}\"),";
if (!string.IsNullOrWhiteSpace(hashes[0]))
crcsha1query += $" (\"{hashes[0]}\", \"{hashes[2]}\"),";
if (!string.IsNullOrWhiteSpace(hashes[1]))
md5sha1query += $" (\"{hashes[1]}\", \"{hashes[2]}\"),";
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
// Now add all of the DAT hashes
// TODO: Do we really need to save the DAT hashes?
sr.Dispose();
}
slc.Dispose();
dbc.Dispose();
}
}
}

145
RombaSharp/Features/Lookup.cs Normal file
View File

@@ -0,0 +1,145 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class Lookup : BaseFeature
{
public const string Value = "Lookup";
public Lookup()
{
Name = Value;
Flags = new List<string>() { "lookup" };
Description = "For each specified hash it looks up any available information.";
_featureType = FeatureType.Flag;
LongDescription = "For each specified hash it looks up any available information (dat or rom).";
Features = new Dictionary<string, Feature>();
AddFeature(SizeInt64Input); // Defaults to -1
AddFeature(OutStringInput);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
long size = GetInt64(features, SizeInt64Value);
string outdat = GetString(features, OutStringValue);
// First, try to figure out what type of hash each is by length and clean it
List<string> crc = new List<string>();
List<string> md5 = new List<string>();
List<string> sha1 = new List<string>();
foreach (string input in Inputs)
{
string temp = string.Empty;
if (input.Length == Constants.CRCLength)
{
temp = Sanitizer.CleanCRC32(input);
if (!string.IsNullOrWhiteSpace(temp))
{
crc.Add(temp);
}
}
else if (input.Length == Constants.MD5Length)
{
temp = Sanitizer.CleanMD5(input);
if (!string.IsNullOrWhiteSpace(temp))
{
md5.Add(temp);
}
}
else if (input.Length == Constants.SHA1Length)
{
temp = Sanitizer.CleanSHA1(input);
if (!string.IsNullOrWhiteSpace(temp))
{
sha1.Add(temp);
}
}
}
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now, search for each of them and return true or false for each
foreach (string input in crc)
{
string query = $"SELECT * FROM crc WHERE crc=\"{input}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
int count = 0;
while (sldr.Read())
{
count++;
}
Globals.Logger.User($"For hash '{input}' there were {count} matches in the database");
}
else
{
Globals.Logger.User($"Hash '{input}' had no matches in the database");
}
sldr.Dispose();
slc.Dispose();
}
foreach (string input in md5)
{
string query = $"SELECT * FROM md5 WHERE md5=\"{input}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
int count = 0;
while (sldr.Read())
{
count++;
}
Globals.Logger.User($"For hash '{input}' there were {count} matches in the database");
}
else
{
Globals.Logger.User($"Hash '{input}' had no matches in the database");
}
sldr.Dispose();
slc.Dispose();
}
foreach (string input in sha1)
{
string query = $"SELECT * FROM sha1 WHERE sha1=\"{input}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
int count = 0;
while (sldr.Read())
{
count++;
}
Globals.Logger.User($"For hash '{input}' there were {count} matches in the database");
}
else
{
Globals.Logger.User($"Hash '{input}' had no matches in the database");
}
sldr.Dispose();
slc.Dispose();
}
dbc.Dispose();
}
}
}

28
RombaSharp/Features/Memstats.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Memstats : BaseFeature
{
public const string Value = "Memstats";
public Memstats()
{
Name = Value;
Flags = new List<string>() { "memstats" };
Description = "Prints memory stats.";
_featureType = FeatureType.Flag;
LongDescription = "Print memory stats.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.User("This feature is not yet implemented: memstats");
}
}
}

77
RombaSharp/Features/Merge.cs Normal file
View File

@@ -0,0 +1,77 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class Merge : BaseFeature
{
public const string Value = "Merge";
public Merge()
{
Name = Value;
Flags = new List<string>() { "merge" };
Description = "Merges depot";
_featureType = FeatureType.Flag;
LongDescription = "Merges specified depot into current depot.";
Features = new Dictionary<string, Feature>();
AddFeature(OnlyNeededFlag);
AddFeature(ResumeStringInput);
AddFeature(WorkersInt32Input);
AddFeature(SkipInitialScanFlag);
}
// TODO: Add way of specifying "current depot" since that's what Romba relies on
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
bool skipInitialscan = GetBoolean(features, SkipInitialScanValue);
int workers = GetInt32(features, WorkersInt32Value);
string resume = GetString(features, ResumeStringValue);
Globals.Logger.Error("This feature is not yet implemented: merge");
// Verify that the inputs are valid directories
Inputs = DirectoryExtensions.GetDirectoriesOnly(Inputs).Select(p => p.CurrentPath).ToList();
// Loop over all input directories
foreach (string input in Inputs)
{
List<string> depotFiles = Directory.EnumerateFiles(input, "*.gz", SearchOption.AllDirectories).ToList();
// If we are copying all that is possible but we want to scan first
if (!onlyNeeded && !skipInitialscan)
{
}
// If we are copying all that is possible but we don't care to scan first
else if (!onlyNeeded && skipInitialscan)
{
}
// If we are copying only what is needed but we want to scan first
else if (onlyNeeded && !skipInitialscan)
{
}
// If we are copying only what is needed but we don't care to scan first
else if (onlyNeeded && skipInitialscan)
{
}
}
}
}
}

49
RombaSharp/Features/Miss.cs Normal file
View File

@@ -0,0 +1,49 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
namespace RombaSharp.Features
{
internal class Miss : BaseFeature
{
public const string Value = "Miss";
// Unique to RombaSharp
public Miss()
{
Name = Value;
Flags = new List<string>() { "miss" };
Description = "Create miss and have file";
_featureType = FeatureType.Flag;
LongDescription = "For each specified DAT file, create miss and have file";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Verify the filenames
Dictionary<string, string> foundDats = GetValidDats(Inputs);
// Create the new output directory if it doesn't exist
DirectoryExtensions.Ensure(Path.Combine(Globals.ExeDir, "out"), create: true);
// Now that we have the dictionary, we can loop through and output to a new folder for each
foreach (string key in foundDats.Keys)
{
// Get the DAT file associated with the key
DatFile datFile = DatFile.CreateAndParse(Path.Combine(_dats, foundDats[key]));
// Now loop through and see if all of the hash combinations exist in the database
/* ended here */
}
Globals.Logger.Error("This feature is not yet implemented: miss");
}
}
}

28
RombaSharp/Features/Progress.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Progress : BaseFeature
{
public const string Value = "Progress";
public Progress()
{
Name = Value;
Flags = new List<string>() { "progress" };
Description = "Shows progress of the currently running command.";
_featureType = FeatureType.Flag;
LongDescription = "Shows progress of the currently running command.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.User("This feature is not yet implemented: progress");
}
}
}

46
RombaSharp/Features/PurgeBackup.cs Normal file
View File

@@ -0,0 +1,46 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class PurgeBackup : BaseFeature
{
public const string Value = "Purge Backup";
public PurgeBackup()
{
Name = Value;
Flags = new List<string>() { "purge-backup" };
Description = "Moves DAT index entries for orphaned DATs.";
_featureType = FeatureType.Flag;
LongDescription = @"Deletes DAT index entries for orphaned DATs and moves ROM files that are no
longer associated with any current DATs to the specified backup folder.
The files will be placed in the backup location using
a folder structure according to the original DAT master directory tree
structure. It also deletes the specified DATs from the DAT index.";
Features = new Dictionary<string, Feature>();
AddFeature(BackupStringInput);
AddFeature(WorkersInt32Input);
AddFeature(DepotListStringInput);
AddFeature(DatsListStringInput);
AddFeature(LogOnlyFlag);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool logOnly = GetBoolean(features, LogOnlyValue);
int workers = GetInt32(features, WorkersInt32Value);
string backup = GetString(features, BackupStringValue);
List<string> dats = GetList(features, DatsListStringValue);
List<string> depot = GetList(features, DepotListStringValue);
Globals.Logger.Error("This feature is not yet implemented: purge-backup");
}
}
}

45
RombaSharp/Features/PurgeDelete.cs Normal file
View File

@@ -0,0 +1,45 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class PurgeDelete : BaseFeature
{
public const string Value = "Purge Delete";
// Unique to RombaSharp
public PurgeDelete()
{
Name = Value;
Flags = new List<string>() { "purge-delete" };
Description = "Deletes DAT index entries for orphaned DATs";
_featureType = FeatureType.Flag;
LongDescription = @"Deletes DAT index entries for orphaned DATs and moves ROM files that are no
longer associated with any current DATs to the specified backup folder.
The files will be placed in the backup location using
a folder structure according to the original DAT master directory tree
structure. It also deletes the specified DATs from the DAT index.";
Features = new Dictionary<string, Feature>();
AddFeature(WorkersInt32Input);
AddFeature(DepotListStringInput);
AddFeature(DatsListStringInput);
AddFeature(LogOnlyFlag);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool logOnly = GetBoolean(features, LogOnlyValue);
int workers = GetInt32(features, WorkersInt32Value);
List<string> dats = GetList(features, DatsListStringValue);
List<string> depot = GetList(features, DepotListStringValue);
Globals.Logger.Error("This feature is not yet implemented: purge-delete");
}
}
}

139
RombaSharp/Features/RefreshDats.cs Normal file
View File

@@ -0,0 +1,139 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class RefreshDats : BaseFeature
{
public const string Value = "Refresh DATs";
public RefreshDats()
{
Name = Value;
Flags = new List<string>() { "refresh-dats" };
Description = "Refreshes the DAT index from the files in the DAT master directory tree.";
_featureType = FeatureType.Flag;
LongDescription = @"Refreshes the DAT index from the files in the DAT master directory tree.
Detects any changes in the DAT master directory tree and updates the DAT index
accordingly, marking deleted or overwritten dats as orphaned and updating
contents of any changed dats.";
Features = new Dictionary<string, Feature>();
AddFeature(WorkersInt32Input);
AddFeature(MissingSha1sStringInput);
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
int workers = GetInt32(features, WorkersInt32Value);
string missingSha1s = GetString(features, MissingSha1sStringValue);
// Make sure the db is set
if (string.IsNullOrWhiteSpace(_db))
{
_db = "db.sqlite";
_connectionString = $"Data Source={_db};Version = 3;";
}
// Make sure the file exists
if (!File.Exists(_db))
DatabaseTools.EnsureDatabase(_dbSchema, _db, _connectionString);
// Make sure the dats dir is set
if (string.IsNullOrWhiteSpace(_dats))
_dats = "dats";
_dats = Path.Combine(Globals.ExeDir, _dats);
// Make sure the folder exists
if (!Directory.Exists(_dats))
Directory.CreateDirectory(_dats);
// First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create();
datroot.Header.Type = "SuperDAT";
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
datroot.PopulateFromDir(_dats, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null);
datroot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)
List<string> databaseDats = new List<string>();
List<string> unneeded = new List<string>();
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Populate the List from the database
InternalStopwatch watch = new InternalStopwatch("Populating the list of existing DATs");
string query = "SELECT DISTINCT hash FROM dat";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
sldr.Read();
string hash = sldr.GetString(0);
if (datroot.Items.ContainsKey(hash))
{
datroot.Items.Remove(hash);
databaseDats.Add(hash);
}
else if (!databaseDats.Contains(hash))
{
unneeded.Add(hash);
}
}
datroot.Items.BucketBy(BucketedBy.Game, DedupeType.None, norename: true);
watch.Stop();
slc.Dispose();
sldr.Dispose();
// Loop through the Dictionary and add all data
watch.Start("Adding new DAT information");
foreach (string key in datroot.Items.Keys)
{
foreach (Rom value in datroot.Items[key])
{
AddDatToDatabase(value, dbc);
}
}
watch.Stop();
// Now loop through and remove all references to old Dats
if (unneeded.Count > 0)
{
watch.Start("Removing unmatched DAT information");
query = "DELETE FROM dat WHERE";
foreach (string dathash in unneeded)
{
query += $" OR hash=\"{dathash}\"";
}
query = query.Replace("WHERE OR", "WHERE");
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
watch.Stop();
}
dbc.Dispose();
}
}
}

166
RombaSharp/Features/RescanDepots.cs Normal file
View File

@@ -0,0 +1,166 @@
using System.Collections.Generic;
using System.IO;
using SabreTools.Library.Data;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using SabreTools.Library.Help;
using Microsoft.Data.Sqlite;
namespace RombaSharp.Features
{
internal class RescanDepots : BaseFeature
{
public const string Value = "Rescan Depots";
// Unique to RombaSharp
public RescanDepots()
{
Name = Value;
Flags = new List<string>() { "depot-rescan" };
Description = "Rescan a specific depot to get new information";
_featureType = FeatureType.Flag;
LongDescription = "Rescan a specific depot to get new information";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.Error("This feature is not yet implemented: rescan-depots");
foreach (string depotname in Inputs)
{
// Check that it's a valid depot first
if (!_depots.ContainsKey(depotname))
{
Globals.Logger.User($"'{depotname}' is not a recognized depot. Please add it to your configuration file and try again");
return;
}
// Then check that the depot is online
if (!Directory.Exists(depotname))
{
Globals.Logger.User($"'{depotname}' does not appear to be online. Please check its status and try again");
return;
}
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// If we have it, then check for all hashes that are in that depot
List<string> hashes = new List<string>();
string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
hashes.Add(sldr.GetString(0));
}
}
// Now rescan the depot itself
DatFile depot = DatFile.Create();
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
depot.PopulateFromDir(depotname, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null);
depot.Items.BucketBy(BucketedBy.SHA1, DedupeType.None);
// Set the base queries to use
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Once we have both, check for any new files
List<string> dupehashes = new List<string>();
IEnumerable<string> keys = depot.Items.Keys;
foreach (string key in keys)
{
List<DatItem> roms = depot.Items[key];
foreach (Rom rom in roms)
{
if (hashes.Contains(rom.SHA1))
{
dupehashes.Add(rom.SHA1);
hashes.Remove(rom.SHA1);
}
else if (!dupehashes.Contains(rom.SHA1))
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcquery += $" (\"{rom.CRC}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5query += $" (\"{rom.MD5}\"),";
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{depotname}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
if (!string.IsNullOrWhiteSpace(rom.MD5))
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
// Now that we've added the information, we get to remove all of the hashes that we want to
query = @"DELETE FROM sha1
JOIN crcsha1
ON sha1.sha1=crcsha1.sha1
JOIN md5sha1
ON sha1.sha1=md5sha1.sha1
JOIN crc
ON crcsha1.crc=crc.crc
JOIN md5
ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN ";
query += $"({string.Join("\",\"", hashes)}\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
// Dispose of the database connection
slc.Dispose();
dbc.Dispose();
}
}
}
}

22
RombaSharp/Features/Script.cs Normal file
View File

@@ -0,0 +1,22 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Script : BaseFeature
{
public const string Value = "Script";
public Script()
{
Name = Value;
Flags = new List<string>() { "--script" };
Description = "Enable script mode (no clear screen)";
_featureType = FeatureType.Flag;
LongDescription = "For times when RombaSharp is being used in a scripted environment, the user may not want the screen to be cleared every time that it is called. This flag allows the user to skip clearing the screen on run just like if the console was being redirected.";
Features = new Dictionary<string, Feature>();
}
}
}

28
RombaSharp/Features/Shutdown.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Shutdown : BaseFeature
{
public const string Value = "Shutdown";
public Shutdown()
{
Name = Value;
Flags = new List<string>() { "shutdown" };
Description = "Gracefully shuts down server.";
_featureType = FeatureType.Flag;
LongDescription = "Gracefully shuts down server saving all the cached data.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.User("This feature is not yet implemented: shutdown");
}
}
}

28
RombaSharp/Features/Version.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
namespace RombaSharp.Features
{
internal class Version : BaseFeature
{
public const string Value = "Version";
public Version()
{
Name = Value;
Flags = new List<string>() { "version" };
Description = "Prints version";
_featureType = FeatureType.Flag;
LongDescription = "Prints version.";
Features = new Dictionary<string, Feature>();
}
public override void ProcessFeatures(Dictionary<string, Feature> features)
{
base.ProcessFeatures(features);
Globals.Logger.User($"RombaSharp version: {Constants.Version}");
}
}
}

140
RombaSharp/RombaSharp.cs → RombaSharp/Program.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using RombaSharp.Features;
using SabreTools.Library.Data;
using SabreTools.Library.Help;
using SabreTools.Library.Tools;
@@ -15,30 +16,8 @@ namespace RombaSharp
/// that needs to read from the depot themselves, if the depot folder cannot be found, the
/// user is prompted to reconnect the depot OR skip that depot entirely.
/// </remarks>
public partial class RombaSharp
public class Program
{
// General settings
private static string _logdir; // Log folder location
private static string _tmpdir; // Temp folder location
private static string _webdir; // Web frontend location
private static string _baddir; // Fail-to-unpack file folder location
private static int _verbosity; // Verbosity of the output
private static int _cores; // Forced CPU cores
// DatRoot settings
private static string _dats; // DatRoot folder location
private static string _db; // Database name
// Depot settings
private static Dictionary<string, Tuple<long, bool>> _depots; // Folder location, Max size
// Server settings
private static int _port; // Web server port
// Other private variables
private const string _config = "config.xml";
private const string _dbSchema = "rombasharp";
private static string _connectionString;
private static Help _help;
/// <summary>
@@ -49,11 +28,8 @@ namespace RombaSharp
// Perform initial setup and verification
Globals.Logger = new Logger(true, "romba.log");
InitializeConfiguration();
DatabaseTools.EnsureDatabase(_dbSchema, _db, _connectionString);
// Create a new Help object for this program
_help = RombaSharp.RetrieveHelp();
_help = RetrieveHelp();
// Get the location of the script tag, if it exists
int scriptLocation = (new List<string>(args)).IndexOf("--script");
@@ -62,7 +38,7 @@ namespace RombaSharp
if (!Console.IsOutputRedirected && scriptLocation == -1)
{
Console.Clear();
Build.PrepareConsole("RombaSharp");
SabreTools.Library.Data.Build.PrepareConsole("RombaSharp");
}
// Now we remove the script tag because it messes things up
@@ -105,10 +81,10 @@ namespace RombaSharp
featureName = _help.GetFeatureName(featureName);
// Get the associated feature
RombaSharpFeature feature = _help[featureName] as RombaSharpFeature;
BaseFeature feature = _help[featureName] as BaseFeature;
// If we had the help feature first
if (featureName == HelpFeature.Value || featureName == DetailedHelpFeature.Value)
if (featureName == DisplayHelp.Value || featureName == DisplayHelpDetailed.Value)
{
feature.ProcessArgs(args, _help);
Globals.Logger.Close();
@@ -126,40 +102,40 @@ namespace RombaSharp
Dictionary<string, Feature> features = _help.GetEnabledFeatures();
switch (featureName)
{
case DetailedHelpFeature.Value:
case HelpFeature.Value:
case ScriptFeature.Value:
case DisplayHelpDetailed.Value:
case DisplayHelp.Value:
case Script.Value:
// No-op as this should be caught
break;
// Require input verification
case ArchiveFeature.Value:
case BuildFeature.Value:
case DatStatsFeature.Value:
case FixdatFeature.Value:
case ImportFeature.Value:
case LookupFeature.Value:
case MergeFeature.Value:
case MissFeature.Value:
case RescanDepotsFeature.Value:
case Archive.Value:
case Features.Build.Value:
case DatStats.Value:
case Fixdat.Value:
case Import.Value:
case Lookup.Value:
case Merge.Value:
case Miss.Value:
case RescanDepots.Value:
VerifyInputs(feature.Inputs, featureName);
feature.ProcessFeatures(features);
break;
// Requires no input verification
case CancelFeature.Value:
case DbStatsFeature.Value:
case DiffdatFeature.Value:
case Dir2DatFeature.Value:
case EDiffdatFeature.Value:
case ExportFeature.Value:
case MemstatsFeature.Value:
case ProgressFeature.Value:
case PurgeBackupFeature.Value:
case PurgeDeleteFeature.Value:
case RefreshDatsFeature.Value:
case ShutdownFeature.Value:
case VersionFeature.Value:
case Cancel.Value:
case DbStats.Value:
case Diffdat.Value:
case Dir2Dat.Value:
case EDiffdat.Value:
case Export.Value:
case Memstats.Value:
case Progress.Value:
case PurgeBackup.Value:
case PurgeDelete.Value:
case RefreshDats.Value:
case Shutdown.Value:
case Features.Version.Value:
feature.ProcessFeatures(features);
break;
@@ -173,6 +149,60 @@ namespace RombaSharp
return;
}
/// <summary>
/// Generate a Help object for this program
/// </summary>
/// <returns>Populated Help object</returns>
private static Help RetrieveHelp()
{
// Create and add the header to the Help object
string barrier = "-----------------------------------------";
List<string> helpHeader = new List<string>()
{
"RombaSharp - C# port of the Romba rom management tool",
barrier,
"Usage: RombaSharp [option] [filename|dirname] ...",
string.Empty
};
// Create the base help object with header
Help help = new Help(helpHeader);
// Add all of the features
help.Add(new DisplayHelp());
help.Add(new DisplayHelpDetailed());
help.Add(new Script());
help.Add(new Archive());
help.Add(new Features.Build());
help.Add(new Cancel());
help.Add(new DatStats());
help.Add(new DbStats());
help.Add(new Diffdat());
help.Add(new Dir2Dat());
help.Add(new EDiffdat());
help.Add(new Export());
help.Add(new Fixdat());
help.Add(new Import());
help.Add(new Lookup());
help.Add(new Memstats());
help.Add(new Merge());
help.Add(new Miss());
help.Add(new PurgeBackup());
help.Add(new PurgeDelete());
help.Add(new RefreshDats());
help.Add(new RescanDepots());
help.Add(new Progress());
help.Add(new Shutdown());
help.Add(new Features.Version());
return help;
}
/// <summary>
/// Verify that there are inputs, show help otherwise
/// </summary>
/// <param name="inputs">List of inputs</param>
/// <param name="feature">Name of the current feature</param>
private static void VerifyInputs(List<string> inputs, string feature)
{
if (inputs.Count == 0)

1854
RombaSharp/RombaSharp.Help.cs
View File

File diff suppressed because it is too large Load Diff

2
SabreTools/Program.cs
View File

@@ -132,7 +132,7 @@ namespace SabreTools
/// <summary>
/// Generate a Help object for this program
/// </summary>
/// <returns></returns>
/// <returns>Populated Help object</returns>
private static Help RetrieveHelp()
{
// Create and add the header to the Help object