claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[RombaSharp] Progress commit with a bunch of things

Browse Source
This commit is contained in:
Matt Nadareski
2016-10-19 10:47:23 -07:00
parent bb48d1bc6a
commit d1697bbef8
4 changed files with 122 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
RombaSharp/Partials/RombaSharp_Helpers.cs
View File

@@ -387,8 +387,14 @@ namespace SabreTools
Directory.CreateDirectory(_dats);
}
// First get a list of SHA-1's from the input DATs
DatFile datroot = new DatFile { Type = "SuperDAT", };
datroot.PopulateDatFromDir(_dats, false, false, false, false, false, false, false, _tmpdir, false, null, 4, _logger);
datroot.BucketBySHA1(false, _logger, false);
// Create a List of dat hashes in the database (SHA-1)
List<string> databaseDats = new List<string>();
List<string> unneeded = new List<string>();
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
@@ -404,53 +410,64 @@ namespace SabreTools
{
sldr.Read();
string hash = sldr.GetString(0);
if (!databaseDats.Contains(hash))
if (datroot.Files.ContainsKey(hash))
{
datroot.Files[hash] = null;
databaseDats.Add(hash);
}
else if (!databaseDats.Contains(hash))
{
unneeded.Add(hash);
}
}
datroot.BucketByGame(false, true, _logger, false);
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
slc.Dispose();
sldr.Dispose();
// Now create a Dictionary of dats to parse from what's not in the database (SHA-1, Path)
Dictionary<string, string> toscan = new Dictionary<string, string>();
// Loop through the datroot and add only needed files
_logger.User("Scanning DAT folder: '" + _dats + "'");
start = DateTime.Now;
foreach (string file in Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories))
{
Rom dat = FileTools.GetFileInfo(file, _logger);
// If the Dat isn't in the database and isn't already accounted for in the DatRoot, add it
if (!databaseDats.Contains(dat.SHA1) && !toscan.ContainsKey(dat.SHA1))
{
toscan.Add(dat.SHA1, Path.GetFullPath(file));
}
// If the Dat is in the database already, remove it to find stragglers
else if (databaseDats.Contains(dat.SHA1))
{
databaseDats.Remove(dat.SHA1);
}
}
_logger.User("Scanning complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Loop through the Dictionary and add all data
_logger.User("Adding new DAT information");
start = DateTime.Now;
foreach (string key in toscan.Keys)
foreach (string key in datroot.Files.Keys)
{
foreach (Rom value in datroot.Files[key])
{
AddDatToDatabase(value, dbc);
}
}
_logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Now loop through and remove all references to old Dats
_logger.User("Removing unmatched DAT information");
start = DateTime.Now;
foreach (string dathash in unneeded)
{
query = "DELETE FROM dats WHERE hash=\"" + dathash + "\"";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
_logger.User("Removing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
dbc.Dispose();
}
private static void AddDatToDatabase(Rom dat, SqliteConnection dbc)
{
// Get the dat full path
string fullpath = Path.Combine(_dats, (dat.MachineName == "dats" ? "" : dat.MachineName), dat.Name);
// Parse the Dat if possible
_logger.User("Adding from '" + toscan[key] + "'");
_logger.User("Adding from '" + dat.Name + "'");
DatFile tempdat = new DatFile();
tempdat.Parse(toscan[key], 0, 0, _logger);
tempdat.Parse(fullpath, 0, 0, _logger);
// If the Dat wasn't empty, add the information
SqliteCommand slc = new SqliteCommand();
if (tempdat.Files.Count != 0)
{
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
@@ -462,9 +479,13 @@ namespace SabreTools
// Loop through the parsed entries
foreach (string romkey in tempdat.Files.Keys)
{
foreach (Rom rom in tempdat.Files[romkey])
foreach (DatItem datItem in tempdat.Files[romkey])
{
_logger.Verbose("Checking and adding file '" + rom.Name);
_logger.Verbose("Checking and adding file '" + datItem.Name);
if (datItem.Type == ItemType.Rom)
{
Rom rom = (Rom)datItem;
if (!String.IsNullOrEmpty(rom.CRC))
{
@@ -488,6 +509,25 @@ namespace SabreTools
}
}
}
else if (datItem.Type == ItemType.Disk)
{
Disk disk = (Disk)datItem;
if (!String.IsNullOrEmpty(disk.MD5))
{
md5query += " (\"" + disk.MD5 + "\"),";
}
if (!String.IsNullOrEmpty(disk.SHA1))
{
sha1query += " (\"" + disk.SHA1 + "\"),";
if (!String.IsNullOrEmpty(disk.MD5))
{
md5sha1query += " (\"" + disk.MD5 + "\", \"" + disk.SHA1 + "\"),";
}
}
}
}
}
// Now run the queries after fixing them
@@ -517,25 +557,12 @@ namespace SabreTools
slc.ExecuteNonQuery();
}
}
}
_logger.User("Adding complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Now loop through and remove all references to old Dats
// TODO: Remove orphaned files as well
_logger.User("Removing unmatched DAT information");
start = DateTime.Now;
foreach (string dathash in databaseDats)
{
query = "DELETE FROM dats WHERE hash=\"" + dathash + "\"";
slc = new SqliteCommand(query, dbc);
string datquery = "INSERT OR IGNORE INTO dat (hash) VALUES (\"" + dat.SHA1 + "\")";
slc = new SqliteCommand(datquery, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
_logger.User("Removing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
dbc.Dispose();
}
/// <summary>
/// Rescan a particular depot path into the database
@@ -576,7 +603,7 @@ namespace SabreTools
// Now rescan the depot itself
DatFile depot = new DatFile();
depot.PopulateDatFromDir(depotname, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger);
depot.PopulateDatFromDir(depotname, false, false, false, false, true, false, false, _tmpdir, false, null, _workers, _logger);
depot.BucketBySHA1(false, _logger, false);
// Set the base queries to use

4
RombaSharp/Partials/RombaSharp_Inits.cs
View File

@@ -32,7 +32,7 @@ namespace SabreTools
DatFile df = new DatFile();
foreach (string dir in onlyDirs)
{
df.PopulateDatFromDir(dir, false, false, false, false, true, false, false, _tmpdir, false, null, 4, _logger);
df.PopulateDatFromDir(dir, false, false, false, false, true, false, false, _tmpdir, false, null, _workers, _logger);
}
// Create an empty Dat for files that need to be rebuilt
@@ -255,7 +255,7 @@ namespace SabreTools
{
datdata.PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */,
true /* enableGzip */, false /* addBlanks */, false /* addDate */, _tmpdir /* tempDir */, false /* copyFiles */,
null /* headerToCheckAgainst */, 4 /* maxDegreeOfParallelism */, _logger);
null /* headerToCheckAgainst */, _workers /* maxDegreeOfParallelism */, _logger);
datdata.WriteToFile("", logger);
}
logger.Close();

5
RombaSharp/RombaSharp.cs
View File

@@ -1,6 +1,6 @@
using System;
using Mono.Data.Sqlite;
using System;
using System.Collections.Generic;
using System.IO;
using SabreTools.Helper;
namespace SabreTools
@@ -22,6 +22,7 @@ namespace SabreTools
// DatRoot settings
private static string _dats; // DatRoot folder location
private static string _db; // Database name
private static SqliteConnection _dbc;
// Depot settings
private static Dictionary<string, Tuple<long, bool>> _depots; // Folder location, Max size

4
SabreTools.Helper/Objects/SimpleSort.cs
View File

@@ -1086,11 +1086,11 @@ namespace SabreTools.Helper
if (_tgz)
{
success &= ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger);
success &= ArchiveTools.WriteTorrentGZ(file, _outDir, _romba, _logger);
}
else
{
success &= ArchiveTools.WriteToArchive(input, _outDir, rom, _logger);
success &= ArchiveTools.WriteToArchive(file, _outDir, rom, _logger);
}
}