claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduce filter-on-read as first defense

Browse Source
This commit is contained in:
Matt Nadareski
2025-05-02 22:34:38 -04:00
parent 1d2e99b246
commit 8b618cea48
20 changed files with 449 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
SabreTools.DatFiles/Formats/ClrMamePro.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using SabreTools.Core.Filter;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
@@ -43,7 +44,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
try
{
@@ -52,7 +58,7 @@ namespace SabreTools.DatFiles.Formats
var metadata = new Serialization.CrossModel.ClrMamePro().Serialize(metadataFile);
// Convert to the internal format
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly);
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly, filterRunner);
}
catch (Exception ex) when (!throwOnError)
{

10
SabreTools.DatFiles/Formats/Hashfile.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using SabreTools.Core.Filter;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
using SabreTools.Hashing;
@@ -27,7 +28,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
try
{
@@ -36,7 +42,7 @@ namespace SabreTools.DatFiles.Formats
var metadata = new Serialization.CrossModel.Hashfile().Serialize(hashfile);
// Convert to the internal format
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly);
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly, filterRunner);
}
catch (Exception ex) when (!throwOnError)
{

10
SabreTools.DatFiles/Formats/Listxml.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using SabreTools.Core.Filter;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
@@ -220,7 +221,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
try
{
@@ -238,7 +244,7 @@ namespace SabreTools.DatFiles.Formats
}
// Convert to the internal format
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly);
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly, filterRunner);
}
catch (Exception ex) when (!throwOnError)
{

8
SabreTools.DatFiles/Formats/Missfile.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using SabreTools.Core.Filter;
using SabreTools.DatItems;
namespace SabreTools.DatFiles.Formats
@@ -26,7 +27,12 @@ namespace SabreTools.DatFiles.Formats
/// <inheritdoc/>
/// <remarks>There is no consistent way to parse a missfile</remarks>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
throw new NotImplementedException();
}

46
SabreTools.DatFiles/Formats/SabreJSON.cs
View File

@@ -6,6 +6,7 @@ using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using SabreTools.Core.Filter;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
@@ -32,7 +33,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
// Prepare all internal variables
var fs = System.IO.File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
@@ -68,7 +74,7 @@ namespace SabreTools.DatFiles.Formats
// Machine array
case "machines":
ReadMachines(jtr, statsOnly, source, sourceIndex: 0);
ReadMachines(jtr, statsOnly, source, sourceIndex: 0, filterRunner);
jtr.Read();
break;
@@ -110,7 +116,8 @@ namespace SabreTools.DatFiles.Formats
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="source">Source representing the DAT</param>
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
private void ReadMachines(JsonTextReader jtr, bool statsOnly, Source source, long sourceIndex)
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadMachines(JsonTextReader jtr, bool statsOnly, Source source, long sourceIndex, FilterRunner? filterRunner)
{
// If the reader is invalid, skip
if (jtr == null)
@@ -124,7 +131,7 @@ namespace SabreTools.DatFiles.Formats
// Loop through each machine object and process
foreach (JObject machineObj in machineArray)
{
ReadMachine(machineObj, statsOnly, source, sourceIndex);
ReadMachine(machineObj, statsOnly, source, sourceIndex, filterRunner);
}
}
@@ -135,7 +142,8 @@ namespace SabreTools.DatFiles.Formats
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="source">Source representing the DAT</param>
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
private void ReadMachine(JObject machineObj, bool statsOnly, Source source, long sourceIndex)
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadMachine(JObject machineObj, bool statsOnly, Source source, long sourceIndex, FilterRunner? filterRunner)
{
// If object is invalid, skip it
if (machineObj == null)
@@ -148,6 +156,10 @@ namespace SabreTools.DatFiles.Formats
if (machineObj.ContainsKey("machine"))
machine = machineObj["machine"]?.ToObject<Machine>();
// If the machine doesn't pass the filter
if (machine != null && filterRunner != null && !machine.PassesFilter(filterRunner))
return;
// Add the machine to the dictionary
// long machineIndex = -1;
// if (machine != null)
@@ -155,7 +167,15 @@ namespace SabreTools.DatFiles.Formats
// Read items, if possible
if (machineObj.ContainsKey("items"))
ReadItems(machineObj["items"] as JArray, statsOnly, source, sourceIndex, machine, machineIndex: 0);
{
ReadItems(machineObj["items"] as JArray,
statsOnly,
source,
sourceIndex,
machine,
machineIndex: 0,
filterRunner);
}
}
/// <summary>
@@ -167,6 +187,7 @@ namespace SabreTools.DatFiles.Formats
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
/// <param name="machine">Machine information to add to the parsed items</param>
/// <param name="machineIndex">Index of the Machine to add to the parsed items</param>
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadItems(
JArray? itemsArr,
bool statsOnly,
@@ -177,7 +198,8 @@ namespace SabreTools.DatFiles.Formats
// Miscellaneous
Machine? machine,
long machineIndex)
long machineIndex,
FilterRunner? filterRunner)
{
// If the array is invalid, skip
if (itemsArr == null)
@@ -186,7 +208,7 @@ namespace SabreTools.DatFiles.Formats
// Loop through each datitem object and process
foreach (JObject itemObj in itemsArr)
{
ReadItem(itemObj, statsOnly, source, sourceIndex, machine, machineIndex);
ReadItem(itemObj, statsOnly, source, sourceIndex, machine, machineIndex, filterRunner);
}
}
@@ -199,6 +221,7 @@ namespace SabreTools.DatFiles.Formats
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
/// <param name="machine">Machine information to add to the parsed items</param>
/// <param name="machineIndex">Index of the Machine to add to the parsed items</param>
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadItem(
JObject itemObj,
bool statsOnly,
@@ -209,7 +232,8 @@ namespace SabreTools.DatFiles.Formats
// Miscellaneous
Machine? machine,
long machineIndex)
long machineIndex,
FilterRunner? filterRunner)
{
// If we have an empty item, skip it
if (itemObj == null)
@@ -359,6 +383,10 @@ namespace SabreTools.DatFiles.Formats
// If we got a valid datitem, copy machine info and add
if (datItem != null)
{
// If the item doesn't pass the filter
if (filterRunner != null && !datItem.PassesFilter(filterRunner))
return;
datItem.CopyMachineInformation(machine);
datItem.SetFieldValue<Source?>(DatItem.SourceKey, source);
AddItem(datItem, statsOnly);

46
SabreTools.DatFiles/Formats/SabreXML.cs
View File

@@ -5,6 +5,7 @@ using System.Text;
using System.Xml;
using System.Xml.Schema;
using System.Xml.Serialization;
using SabreTools.Core.Filter;
using SabreTools.DatItems;
namespace SabreTools.DatFiles.Formats
@@ -29,7 +30,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
// Prepare all internal variables
XmlReader? xtr = XmlReader.Create(filename, new XmlReaderSettings
@@ -73,7 +79,7 @@ namespace SabreTools.DatFiles.Formats
break;
case "directory":
ReadDirectory(xtr.ReadSubtree(), statsOnly, source, sourceIndex);
ReadDirectory(xtr.ReadSubtree(), statsOnly, source, sourceIndex, filterRunner);
// Skip the directory node now that we've processed it
xtr.Read();
@@ -104,7 +110,12 @@ namespace SabreTools.DatFiles.Formats
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="source">Source representing the DAT</param>
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
private void ReadDirectory(XmlReader xtr, bool statsOnly, Source source, long sourceIndex)
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadDirectory(XmlReader xtr,
bool statsOnly,
Source source,
long sourceIndex,
FilterRunner? filterRunner)
{
// If the reader is invalid, skip
if (xtr == null)
@@ -130,6 +141,11 @@ namespace SabreTools.DatFiles.Formats
case "machine":
XmlSerializer xs = new(typeof(Machine));
machine = xs?.Deserialize(xtr.ReadSubtree()) as Machine;
// If the machine doesn't pass the filter
if (machine != null && filterRunner != null && !machine.PassesFilter(filterRunner))
machine = null;
if (machine != null)
machineIndex = AddMachineDB(machine);
@@ -137,7 +153,13 @@ namespace SabreTools.DatFiles.Formats
break;
case "files":
ReadFiles(xtr.ReadSubtree(), machine, machineIndex, statsOnly, source, sourceIndex);
ReadFiles(xtr.ReadSubtree(),
machine,
machineIndex,
statsOnly,
source,
sourceIndex,
filterRunner);
// Skip the directory node now that we've processed it
xtr.Read();
@@ -158,7 +180,14 @@ namespace SabreTools.DatFiles.Formats
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="source">Source representing the DAT</param>
/// <param name="sourceIndex">Index of the Source representing the DAT</param>
private void ReadFiles(XmlReader xtr, Machine? machine, long machineIndex, bool statsOnly, Source source, long sourceIndex)
/// <param name="filterRunner">Optional FilterRunner to filter items on parse</param>
private void ReadFiles(XmlReader xtr,
Machine? machine,
long machineIndex,
bool statsOnly,
Source source,
long sourceIndex,
FilterRunner? filterRunner)
{
// If the reader is invalid, skip
if (xtr == null)
@@ -181,6 +210,13 @@ namespace SabreTools.DatFiles.Formats
XmlSerializer xs = new(typeof(DatItem));
if (xs.Deserialize(xtr.ReadSubtree()) is DatItem item)
{
// If the item doesn't pass the filter
if (filterRunner != null && !item.PassesFilter(filterRunner))
{
xtr.Skip();
break;
}
item.CopyMachineInformation(machine);
item.SetFieldValue<Source?>(DatItem.SourceKey, source);
AddItem(item, statsOnly);

10
SabreTools.DatFiles/Formats/SeparatedValue.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using SabreTools.Core.Filter;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
@@ -36,7 +37,12 @@ namespace SabreTools.DatFiles.Formats
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
try
{
@@ -45,7 +51,7 @@ namespace SabreTools.DatFiles.Formats
var metadata = new Serialization.CrossModel.SeparatedValue().Serialize(metadataFile);
// Convert to the internal format
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly);
ConvertFromMetadata(metadata, filename, indexId, keep, statsOnly, filterRunner);
}
catch (Exception ex) when (!throwOnError)
{

10
SabreTools.DatFiles/Formats/SerializableDatFile.cs
View File

@@ -1,4 +1,5 @@
using System;
using SabreTools.Core.Filter;
using SabreTools.Models.Metadata;
using SabreTools.Serialization.Interfaces;
@@ -20,7 +21,12 @@ namespace SabreTools.DatFiles.Formats
protected SerializableDatFile(DatFile? datFile) : base(datFile) { }
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
public override void ParseFile(string filename,
int indexId,
bool keep,
bool statsOnly = false,
FilterRunner? filterRunner = null,
bool throwOnError = false)
{
try
{
@@ -29,7 +35,7 @@ namespace SabreTools.DatFiles.Formats
var internalFormat = Activator.CreateInstance<TModelSerializer>().Serialize(specificFormat);
// Convert to the internal format
ConvertFromMetadata(internalFormat, filename, indexId, keep, statsOnly);
ConvertFromMetadata(internalFormat, filename, indexId, keep, statsOnly, filterRunner);
}
catch (Exception ex) when (!throwOnError)
{