diff --git a/SabreTools.Core/Prepare.cs b/SabreTools.Core/Prepare.cs index e6f9e664..5fa4beb5 100644 --- a/SabreTools.Core/Prepare.cs +++ b/SabreTools.Core/Prepare.cs @@ -12,8 +12,8 @@ namespace SabreTools.Core /// /// The current toolset version to be used by all child applications /// - public readonly static string Version = $"v1.1.2"; - //public readonly static string Version = $"v1.1.2-{File.GetCreationTime(Assembly.GetExecutingAssembly().Location):yyyy-MM-dd HH:mm:ss}"; + //public readonly static string Version = $"v1.1.2"; + public readonly static string Version = $"v1.1.2-{File.GetCreationTime(Assembly.GetExecutingAssembly().Location):yyyy-MM-dd HH:mm:ss}"; /// /// Readies the console and outputs the header diff --git a/SabreTools.Core/README.1ST b/SabreTools.Core/README.1ST index a340bd01..24106ead 100644 --- a/SabreTools.Core/README.1ST +++ b/SabreTools.Core/README.1ST @@ -290,6 +290,7 @@ Features and Options: Possible values are: all - All available DAT types + ado, archive - Archive.org file list am, attractmode - AttractMode XML cmp, clrmamepro - ClrMamePro csv - Standardized Comma-Separated Value @@ -642,6 +643,7 @@ Features and Options: Possible values are: all - All available DAT types + ado, archive - Archive.org file list am, attractmode - AttractMode XML cmp, clrmamepro - ClrMamePro csv - Standardized Comma-Separated Value @@ -806,6 +808,7 @@ Features and Options: Possible values are: all - All available DAT types + ado, archive - Archive.org file list am, attractmode - AttractMode XML cmp, clrmamepro - ClrMamePro csv - Standardized Comma-Separated Value diff --git a/SabreTools.DatFiles/DatFile.cs b/SabreTools.DatFiles/DatFile.cs index 1522d604..26b81f5d 100644 --- a/SabreTools.DatFiles/DatFile.cs +++ b/SabreTools.DatFiles/DatFile.cs @@ -73,6 +73,7 @@ namespace SabreTools.DatFiles { return datFormat switch { + DatFormat.ArchiveDotOrg => new ArchiveDotOrg(baseDat), DatFormat.AttractMode => new AttractMode(baseDat), DatFormat.ClrMamePro => new ClrMamePro(baseDat, quotes), DatFormat.CSV => new SeparatedValue(baseDat, ','), diff --git a/SabreTools.DatFiles/DatHeader.cs b/SabreTools.DatFiles/DatHeader.cs index 302f623b..bec7542b 100644 --- a/SabreTools.DatFiles/DatHeader.cs +++ b/SabreTools.DatFiles/DatHeader.cs @@ -1029,6 +1029,21 @@ namespace SabreTools.DatFiles } } + // Archive.org + if (DatFormat.HasFlag(DatFormat.ArchiveDotOrg)) + { + if (usedExtensions.Contains(".xml")) + { + outfileNames.Add(DatFormat.ArchiveDotOrg, CreateOutFileNamesHelper(outDir, ".ado.xml", overwrite)); + usedExtensions.Add(".ado.xml"); + } + else + { + outfileNames.Add(DatFormat.ArchiveDotOrg, CreateOutFileNamesHelper(outDir, ".xml", overwrite)); + usedExtensions.Add(".xml"); + } + } + #endregion return outfileNames; diff --git a/SabreTools.DatFiles/Enums.cs b/SabreTools.DatFiles/Enums.cs index 40149df8..1ce78bf3 100644 --- a/SabreTools.DatFiles/Enums.cs +++ b/SabreTools.DatFiles/Enums.cs @@ -45,6 +45,11 @@ namespace SabreTools.DatFiles /// OpenMSX = 1 << 6, + /// + /// Archive.org file list XML + /// + ArchiveDotOrg = 1 << 7, + #endregion #region Propietary Formats @@ -52,22 +57,22 @@ namespace SabreTools.DatFiles /// /// ClrMamePro custom /// - ClrMamePro = 1 << 7, + ClrMamePro = 1 << 8, /// /// RomCenter INI-based /// - RomCenter = 1 << 8, + RomCenter = 1 << 9, /// /// DOSCenter custom /// - DOSCenter = 1 << 9, + DOSCenter = 1 << 10, /// /// AttractMode custom /// - AttractMode = 1 << 10, + AttractMode = 1 << 11, #endregion @@ -76,37 +81,37 @@ namespace SabreTools.DatFiles /// /// ClrMamePro missfile /// - MissFile = 1 << 11, + MissFile = 1 << 12, /// /// Comma-Separated Values (standardized) /// - CSV = 1 << 12, + CSV = 1 << 13, /// /// Semicolon-Separated Values (standardized) /// - SSV = 1 << 13, + SSV = 1 << 14, /// /// Tab-Separated Values (standardized) /// - TSV = 1 << 14, + TSV = 1 << 15, /// /// MAME Listrom output /// - Listrom = 1 << 15, + Listrom = 1 << 16, /// /// Everdrive Packs SMDB /// - EverdriveSMDB = 1 << 16, + EverdriveSMDB = 1 << 17, /// /// SabreJSON /// - SabreJSON = 1 << 17, + SabreJSON = 1 << 18, #endregion @@ -115,37 +120,37 @@ namespace SabreTools.DatFiles /// /// CRC32 hash list /// - RedumpSFV = 1 << 18, + RedumpSFV = 1 << 19, /// /// MD5 hash list /// - RedumpMD5 = 1 << 19, + RedumpMD5 = 1 << 20, /// /// SHA-1 hash list /// - RedumpSHA1 = 1 << 20, + RedumpSHA1 = 1 << 21, /// /// SHA-256 hash list /// - RedumpSHA256 = 1 << 21, + RedumpSHA256 = 1 << 22, /// /// SHA-384 hash list /// - RedumpSHA384 = 1 << 22, + RedumpSHA384 = 1 << 23, /// /// SHA-512 hash list /// - RedumpSHA512 = 1 << 23, + RedumpSHA512 = 1 << 24, /// /// SpamSum hash list /// - RedumpSpamSum = 1 << 24, + RedumpSpamSum = 1 << 25, #endregion diff --git a/SabreTools.DatFiles/Formats/ArchiveDotOrg.cs b/SabreTools.DatFiles/Formats/ArchiveDotOrg.cs new file mode 100644 index 00000000..5e3f6566 --- /dev/null +++ b/SabreTools.DatFiles/Formats/ArchiveDotOrg.cs @@ -0,0 +1,396 @@ +using System; +using System.IO; +using System.Text; +using System.Xml; +using System.Xml.Schema; + +using SabreTools.Core; +using SabreTools.Core.Tools; +using SabreTools.DatItems; +using SabreTools.DatItems.Formats; +using SabreTools.IO; + +namespace SabreTools.DatFiles.Formats +{ + /// + /// Represents parsing and writing of a Archive.org file list + /// + internal class ArchiveDotOrg : DatFile + { + /// + /// Constructor designed for casting a base DatFile + /// + /// Parent DatFile to copy from + public ArchiveDotOrg(DatFile datFile) + : base(datFile) + { + } + + /// + public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false) + { + // Prepare all internal variables + XmlReader xtr = XmlReader.Create(filename, new XmlReaderSettings + { + CheckCharacters = false, + DtdProcessing = DtdProcessing.Ignore, + IgnoreComments = true, + IgnoreWhitespace = true, + ValidationFlags = XmlSchemaValidationFlags.None, + ValidationType = ValidationType.None, + }); + + // If we got a null reader, just return + if (xtr == null) + return; + + // Otherwise, read the file to the end + try + { + xtr.MoveToContent(); + while (!xtr.EOF) + { + // We only want elements + if (xtr.NodeType != XmlNodeType.Element) + { + xtr.Read(); + continue; + } + + switch (xtr.Name) + { + case "files": + ReadFiles(xtr.ReadSubtree(), statsOnly, filename, indexId, keep); + + // Skip the machine now that we've processed it + xtr.Skip(); + break; + + default: + xtr.Read(); + break; + } + } + } + catch (Exception ex) when (!throwOnError) + { + logger.Warning(ex, $"Exception found while parsing '{filename}'"); + + // For XML errors, just skip the affected node + xtr?.Read(); + } + + xtr.Dispose(); + } + + /// + /// Read files information + /// + /// XmlReader to use to parse the machine + /// True to only add item statistics while parsing, false otherwise + /// Name of the file to be parsed + /// Index ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + private void ReadFiles( + XmlReader reader, + bool statsOnly, + + // Standard Dat parsing + string filename, + int indexId, + + // Miscellaneous + bool keep) + { + // If we have an empty machine, skip it + if (reader == null) + return; + + // Otherwise, add what is possible + reader.MoveToContent(); + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the files from the list + switch (reader.Name) + { + case "file": + ReadFile(reader.ReadSubtree(), statsOnly, filename, indexId, keep); + + // Skip the file node now that we've processed it + reader.Skip(); + break; + + default: + reader.Read(); + break; + } + } + } + + /// + /// Read file information + /// + /// XmlReader to use to parse the machine + /// True to only add item statistics while parsing, false otherwise + /// Name of the file to be parsed + /// Index ID for the DAT + /// True if full pathnames are to be kept, false otherwise (default) + private void ReadFile( + XmlReader reader, + bool statsOnly, + + // Standard Dat parsing + string filename, + int indexId, + + // Miscellaneous + bool keep) + { + // If we have an empty machine, skip it + if (reader == null) + return; + + // Otherwise, add what is possible + reader.MoveToContent(); + + // Create the Rom to store the info + Rom rom = new Rom + { + Name = reader.GetAttribute("name"), + Value = reader.GetAttribute("source"), // TODO: Create new field for this + + // TODO: Derive from path, if possible + Machine = new Machine + { + Name = "Default", + Description = "Default", + }, + + Source = new Source + { + Index = indexId, + Name = filename, + } + }; + + // TODO: Handle SuperDAT + //if (Header.Type == "SuperDAT" && !keep) + //{ + // string tempout = Regex.Match(machine.Name, @".*?\\(.*)").Groups[1].Value; + // if (!string.IsNullOrWhiteSpace(tempout)) + // machine.Name = tempout; + //} + + while (!reader.EOF) + { + // We only want elements + if (reader.NodeType != XmlNodeType.Element) + { + reader.Read(); + continue; + } + + // Get the roms from the machine + switch (reader.Name) + { + case "crc32": + rom.CRC = reader.ReadElementContentAsString(); + break; + + case "md5": + rom.MD5 = reader.ReadElementContentAsString(); + break; + + case "mtime": + rom.Date = reader.ReadElementContentAsString(); + break; + + case "sha1": + rom.SHA1 = reader.ReadElementContentAsString(); + break; + + case "size": + rom.Size = Utilities.CleanLong(reader.ReadElementContentAsString()); + break; + + // TODO: Create new field for this + case "format": + string format = reader.ReadElementContentAsString(); + break; + + // TODO: Create new field for this + case "original": + string original = reader.ReadElementContentAsString(); + break; + + // TODO: Create new field for this, Int32? + case "rotation": + string rotation = reader.ReadElementContentAsString(); + break; + + // TODO: Create new field for this + case "summation": + string summation = reader.ReadElementContentAsString(); + break; + + default: + reader.Read(); + break; + } + } + + // Now process and add the rom + ParseAddHelper(rom, statsOnly); + } + + /// + protected override ItemType[] GetSupportedTypes() + { + return new ItemType[] + { + ItemType.Rom, + }; + } + + /// + public override bool WriteToFile(string outfile, bool ignoreblanks = false, bool throwOnError = false) + { + try + { + logger.User($"Writing to '{outfile}'..."); + FileStream fs = File.Create(outfile); + + // If we get back null for some reason, just log and return + if (fs == null) + { + logger.Warning($"File '{outfile}' could not be created for writing! Please check to see if the file is writable"); + return false; + } + + XmlTextWriter xtw = new XmlTextWriter(fs, new UTF8Encoding(false)) + { + Formatting = Formatting.Indented, + IndentChar = '\t', + Indentation = 1 + }; + + // Write out the header + WriteHeader(xtw); + + // Write out each of the machines and roms + string lastgame = null; + + // Use a sorted list of games to output + foreach (string key in Items.SortedKeys) + { + ConcurrentList datItems = Items.FilteredItems(key); + + // If this machine doesn't contain any writable items, skip + if (!ContainsWritable(datItems)) + continue; + + // Resolve the names in the block + datItems = DatItem.ResolveNames(datItems); + + for (int index = 0; index < datItems.Count; index++) + { + DatItem datItem = datItems[index]; + + // Check for a "null" item + datItem = ProcessNullifiedItem(datItem); + + // Write out the item if we're not ignoring + if (!ShouldIgnore(datItem, ignoreblanks)) + WriteDatItem(xtw, datItem); + + // Set the new data to compare against + lastgame = datItem.Machine.Name; + } + } + + // Write the file footer out + WriteFooter(xtw); + + logger.User($"'{outfile}' written!{Environment.NewLine}"); + xtw.Dispose(); + fs.Dispose(); + } + catch (Exception ex) when (!throwOnError) + { + logger.Error(ex); + return false; + } + + return true; + } + + /// + /// Write out DAT header using the supplied StreamWriter + /// + /// XmlTextWriter to output to + private void WriteHeader(XmlTextWriter xtw) + { + xtw.WriteStartDocument(); + + xtw.WriteStartElement("files"); + + xtw.Flush(); + } + + /// + /// Write out DatItem using the supplied StreamWriter + /// + /// XmlTextWriter to output to + /// DatItem object to be output + private void WriteDatItem(XmlTextWriter xtw, DatItem datItem) + { + // Pre-process the item name + ProcessItemName(datItem, true); + + // Build the state + switch (datItem.ItemType) + { + case ItemType.Rom: + var rom = datItem as Rom; + xtw.WriteStartElement("file"); + xtw.WriteOptionalAttributeString("source", rom.Value); + + xtw.WriteOptionalElementString("mtime", rom.Date); + xtw.WriteOptionalElementString("size", rom.Size?.ToString()); + xtw.WriteOptionalElementString("md5", rom.MD5?.ToLowerInvariant()); + xtw.WriteOptionalElementString("crc32", rom.CRC?.ToLowerInvariant()); + xtw.WriteOptionalElementString("sha1", rom.SHA1?.ToLowerInvariant()); + //xtw.WriteOptionalElementString("format", rom.Format); + //xtw.WriteOptionalElementString("original", rom.Original); + //xtw.WriteOptionalElementString("rotation", rom.Rotation?.ToString()); + //xtw.WriteOptionalElementString("summation", rom.Summation); + + // End file + xtw.WriteEndElement(); + break; + } + + xtw.Flush(); + } + + /// + /// Write out DAT footer using the supplied StreamWriter + /// + /// XmlTextWriter to output to + private void WriteFooter(XmlTextWriter xtw) + { + // End files + xtw.WriteEndElement(); + + xtw.Flush(); + } + } +} diff --git a/SabreTools.DatTools/Parser.cs b/SabreTools.DatTools/Parser.cs index 473fdb5f..3cbe3e16 100644 --- a/SabreTools.DatTools/Parser.cs +++ b/SabreTools.DatTools/Parser.cs @@ -199,6 +199,9 @@ namespace SabreTools.DatTools else if ((second.StartsWith(" + + + 1621688177 + 1024 + c41d8cd98f00b204e9800998ecf8427e + deadbeef + ca39a3ee5e6b4b0d3255bfef95601890afd80709 + Unknown + rom2.bin + 0 + md5 + + diff --git a/SabreTools/Features/BaseFeature.cs b/SabreTools/Features/BaseFeature.cs index 5d54c15b..baf4cbae 100644 --- a/SabreTools/Features/BaseFeature.cs +++ b/SabreTools/Features/BaseFeature.cs @@ -1296,6 +1296,7 @@ namespace SabreTools.Features Possible values are: all - All available DAT types + ado, archive - Archive.org file list am, attractmode - AttractMode XML cmp, clrmamepro - ClrMamePro csv - Standardized Comma-Separated Value @@ -2297,6 +2298,9 @@ CREATE TABLE IF NOT EXISTS data ( { case "all": return DatFormat.ALL; + case "ado": + case "archive": + return DatFormat.ArchiveDotOrg; case "am": case "attractmode": return DatFormat.AttractMode;