From 3e5bd7113c22fe04c4ed01163c3985f7402c6d0c Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 8 Sep 2023 17:26:37 -0400 Subject: [PATCH] Add and convert Archive.org --- CrossModel/ArchiveDotOrg.Deserializer.cs | 120 +++++++++++++++++++++ CrossModel/ArchiveDotOrg.Serializer.cs | 131 +++++++++++++++++++++++ Files/ArchiveDotOrg.Deserializer.cs | 7 ++ Streams/ArchiveDotOrg.Deserializer.cs | 7 ++ 4 files changed, 265 insertions(+) create mode 100644 CrossModel/ArchiveDotOrg.Deserializer.cs create mode 100644 CrossModel/ArchiveDotOrg.Serializer.cs create mode 100644 Files/ArchiveDotOrg.Deserializer.cs create mode 100644 Streams/ArchiveDotOrg.Deserializer.cs diff --git a/CrossModel/ArchiveDotOrg.Deserializer.cs b/CrossModel/ArchiveDotOrg.Deserializer.cs new file mode 100644 index 00000000..26c0a3b5 --- /dev/null +++ b/CrossModel/ArchiveDotOrg.Deserializer.cs @@ -0,0 +1,120 @@ +using System; +using System.Linq; +using SabreTools.Models.ArchiveDotOrg; + +namespace SabreTools.Serialization.CrossModel +{ + public partial class ArchiveDotOrg : IModelSerializer + { + /// +#if NET48 + public Models.ArchiveDotOrg.Files Deserialize(Models.Metadata.MetadataFile obj) +#else + public Models.ArchiveDotOrg.Files? Deserialize(Models.Metadata.MetadataFile? obj) +#endif + { + if (obj == null) + return null; + + var files = new Models.ArchiveDotOrg.Files(); + + var machines = obj.Read(Models.Metadata.MetadataFile.MachineKey); + if (machines != null && machines.Any()) + { + files.File = machines + .Where(m => m != null) + .SelectMany(ConvertFromInternalModel) + .ToArray(); + } + + return files; + } + + /// + /// Convert from to an array of + /// + private static Models.ArchiveDotOrg.File[] ConvertFromInternalModel(Models.Metadata.Machine item) + { + var roms = item.Read(Models.Metadata.Machine.RomKey); + if (roms == null) + return Array.Empty(); + + return roms + .Where(r => r != null) + .Select(ConvertFromInternalModel).ToArray(); + } + + /// + /// Convert from to + /// + private static Models.ArchiveDotOrg.File ConvertFromInternalModel(Models.Metadata.Rom item) + { + var file = new Models.ArchiveDotOrg.File + { + Name = item.ReadString(Models.Metadata.Rom.NameKey), + Source = item.ReadString(Models.Metadata.Rom.SourceKey), + BitTorrentMagnetHash = item.ReadString(Models.Metadata.Rom.BitTorrentMagnetHashKey), + LastModifiedTime = item.ReadString(Models.Metadata.Rom.LastModifiedTimeKey), + Size = item.ReadString(Models.Metadata.Rom.SizeKey), + MD5 = item.ReadString(Models.Metadata.Rom.MD5Key), + CRC32 = item.ReadString(Models.Metadata.Rom.CRCKey), + SHA1 = item.ReadString(Models.Metadata.Rom.SHA1Key), + FileCount = item.ReadString(Models.Metadata.Rom.FileCountKey), + Format = item.ReadString(Models.Metadata.Rom.FormatKey), + Original = item.ReadString(Models.Metadata.Rom.OriginalKey), + Summation = item.ReadString(Models.Metadata.Rom.SummationKey), + MatrixNumber = item.ReadString(Models.Metadata.Rom.MatrixNumberKey), + CollectionCatalogNumber = item.ReadString(Models.Metadata.Rom.CollectionCatalogNumberKey), + Comment = item.ReadString(Models.Metadata.Rom.CommentKey), + + ASRDetectedLang = item.ReadString(Models.Metadata.Rom.ASRDetectedLangKey), + ASRDetectedLangConf = item.ReadString(Models.Metadata.Rom.ASRDetectedLangConfKey), + ASRTranscribedLang = item.ReadString(Models.Metadata.Rom.ASRTranscribedLangKey), + WhisperASRModuleVersion = item.ReadString(Models.Metadata.Rom.WhisperASRModuleVersionKey), + WhisperModelHash = item.ReadString(Models.Metadata.Rom.WhisperModelHashKey), + WhisperModelName = item.ReadString(Models.Metadata.Rom.WhisperModelNameKey), + WhisperVersion = item.ReadString(Models.Metadata.Rom.WhisperVersionKey), + + ClothCoverDetectionModuleVersion = item.ReadString(Models.Metadata.Rom.ClothCoverDetectionModuleVersionKey), + hOCRCharToWordhOCRVersion = item.ReadString(Models.Metadata.Rom.hOCRCharToWordhOCRVersionKey), + hOCRCharToWordModuleVersion = item.ReadString(Models.Metadata.Rom.hOCRCharToWordModuleVersionKey), + hOCRFtsTexthOCRVersion = item.ReadString(Models.Metadata.Rom.hOCRFtsTexthOCRVersionKey), + hOCRFtsTextModuleVersion = item.ReadString(Models.Metadata.Rom.hOCRFtsTextModuleVersionKey), + hOCRPageIndexhOCRVersion = item.ReadString(Models.Metadata.Rom.hOCRPageIndexhOCRVersionKey), + hOCRPageIndexModuleVersion = item.ReadString(Models.Metadata.Rom.hOCRPageIndexModuleVersionKey), + TesseractOCR = item.ReadString(Models.Metadata.Rom.TesseractOCRKey), + TesseractOCRConverted = item.ReadString(Models.Metadata.Rom.TesseractOCRConvertedKey), + TesseractOCRDetectedLang = item.ReadString(Models.Metadata.Rom.TesseractOCRDetectedLangKey), + TesseractOCRDetectedLangConf = item.ReadString(Models.Metadata.Rom.TesseractOCRDetectedLangConfKey), + TesseractOCRDetectedScript = item.ReadString(Models.Metadata.Rom.TesseractOCRDetectedScriptKey), + TesseractOCRDetectedScriptConf = item.ReadString(Models.Metadata.Rom.TesseractOCRDetectedScriptConfKey), + TesseractOCRModuleVersion = item.ReadString(Models.Metadata.Rom.TesseractOCRModuleVersionKey), + TesseractOCRParameters = item.ReadString(Models.Metadata.Rom.TesseractOCRParametersKey), + PDFModuleVersion = item.ReadString(Models.Metadata.Rom.PDFModuleVersionKey), + WordConfidenceInterval0To10 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval0To10Key), + WordConfidenceInterval11To20 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval11To20Key), + WordConfidenceInterval21To30 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval21To30Key), + WordConfidenceInterval31To40 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval31To40Key), + WordConfidenceInterval41To50 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval41To50Key), + WordConfidenceInterval51To60 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval51To60Key), + WordConfidenceInterval61To70 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval61To70Key), + WordConfidenceInterval71To80 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval71To80Key), + WordConfidenceInterval81To90 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval81To90Key), + WordConfidenceInterval91To100 = item.ReadString(Models.Metadata.Rom.WordConfidenceInterval91To100Key), + + Album = item.ReadString(Models.Metadata.Rom.AlbumKey), + Artist = item.ReadString(Models.Metadata.Rom.ArtistKey), + Bitrate = item.ReadString(Models.Metadata.Rom.BitrateKey), + Creator = item.ReadString(Models.Metadata.Rom.CreatorKey), + Height = item.ReadString(Models.Metadata.Rom.HeightKey), + Length = item.ReadString(Models.Metadata.Rom.LengthKey), + PreviewImage = item.ReadString(Models.Metadata.Rom.PreviewImageKey), + Rotation = item.ReadString(Models.Metadata.Rom.RotationKey), + Title = item.ReadString(Models.Metadata.Rom.TitleKey), + Track = item.ReadString(Models.Metadata.Rom.TrackKey), + Width = item.ReadString(Models.Metadata.Rom.WidthKey), + }; + return file; + } + } +} \ No newline at end of file diff --git a/CrossModel/ArchiveDotOrg.Serializer.cs b/CrossModel/ArchiveDotOrg.Serializer.cs new file mode 100644 index 00000000..044d7660 --- /dev/null +++ b/CrossModel/ArchiveDotOrg.Serializer.cs @@ -0,0 +1,131 @@ +using System.Linq; + +namespace SabreTools.Serialization.CrossModel +{ + public partial class ArchiveDotOrg : IModelSerializer + { + /// +#if NET48 + public Models.Metadata.MetadataFile Serialize(Models.ArchiveDotOrg.Files item) +#else + public static Models.Metadata.MetadataFile? Serialize(Models.ArchiveDotOrg.Files? item) +#endif + { + if (item == null) + return null; + + var metadataFile = new Models.Metadata.MetadataFile + { + [Models.Metadata.MetadataFile.HeaderKey] = ConvertHeaderToInternalModel(item), + }; + + if (item?.File != null && item.File.Any()) + { + metadataFile[Models.Metadata.MetadataFile.MachineKey] = item.File + .Where(f => f != null) + .Select(ConvertMachineToInternalModel) + .ToArray(); + } + + return metadataFile; + } + + /// + /// Convert from to + /// + private static Models.Metadata.Header ConvertHeaderToInternalModel(Models.ArchiveDotOrg.Files item) + { + var header = new Models.Metadata.Header + { + [Models.Metadata.Header.NameKey] = "archive.org", + }; + return header; + } + + /// + /// Convert from to + /// + private static Models.Metadata.Machine ConvertMachineToInternalModel(Models.ArchiveDotOrg.File item) + { + var machine = new Models.Metadata.Machine + { + [Models.Metadata.Machine.RomKey] = ConvertToInternalModel(item), + }; + return machine; + } + + /// + /// Convert from to + /// + private static Models.Metadata.Rom ConvertToInternalModel(Models.ArchiveDotOrg.File item) + { + var rom = new Models.Metadata.Rom + { + [Models.Metadata.Rom.NameKey] = item.Name, + [Models.Metadata.Rom.SourceKey] = item.Source, + [Models.Metadata.Rom.BitTorrentMagnetHashKey] = item.BitTorrentMagnetHash, + [Models.Metadata.Rom.LastModifiedTimeKey] = item.LastModifiedTime, + [Models.Metadata.Rom.SizeKey] = item.Size, + [Models.Metadata.Rom.MD5Key] = item.MD5, + [Models.Metadata.Rom.CRCKey] = item.CRC32, + [Models.Metadata.Rom.SHA1Key] = item.SHA1, + [Models.Metadata.Rom.FileCountKey] = item.FileCount, + [Models.Metadata.Rom.FormatKey] = item.Format, + [Models.Metadata.Rom.OriginalKey] = item.Original, + [Models.Metadata.Rom.SummationKey] = item.Summation, + [Models.Metadata.Rom.MatrixNumberKey] = item.MatrixNumber, + [Models.Metadata.Rom.CollectionCatalogNumberKey] = item.CollectionCatalogNumber, + [Models.Metadata.Rom.PublisherKey] = item.Publisher, + [Models.Metadata.Rom.CommentKey] = item.Comment, + + [Models.Metadata.Rom.ASRDetectedLangKey] = item.ASRDetectedLang, + [Models.Metadata.Rom.ASRDetectedLangConfKey] = item.ASRDetectedLangConf, + [Models.Metadata.Rom.ASRTranscribedLangKey] = item.ASRTranscribedLang, + [Models.Metadata.Rom.WhisperASRModuleVersionKey] = item.WhisperASRModuleVersion, + [Models.Metadata.Rom.WhisperModelHashKey] = item.WhisperModelHash, + [Models.Metadata.Rom.WhisperModelNameKey] = item.WhisperModelName, + [Models.Metadata.Rom.WhisperVersionKey] = item.WhisperVersion, + + [Models.Metadata.Rom.ClothCoverDetectionModuleVersionKey] = item.ClothCoverDetectionModuleVersion, + [Models.Metadata.Rom.hOCRCharToWordhOCRVersionKey] = item.hOCRCharToWordhOCRVersion, + [Models.Metadata.Rom.hOCRCharToWordModuleVersionKey] = item.hOCRCharToWordModuleVersion, + [Models.Metadata.Rom.hOCRFtsTexthOCRVersionKey] = item.hOCRFtsTexthOCRVersion, + [Models.Metadata.Rom.hOCRFtsTextModuleVersionKey] = item.hOCRFtsTextModuleVersion, + [Models.Metadata.Rom.hOCRPageIndexhOCRVersionKey] = item.hOCRPageIndexhOCRVersion, + [Models.Metadata.Rom.hOCRPageIndexModuleVersionKey] = item.hOCRPageIndexModuleVersion, + [Models.Metadata.Rom.TesseractOCRKey] = item.TesseractOCR, + [Models.Metadata.Rom.TesseractOCRConvertedKey] = item.TesseractOCRConverted, + [Models.Metadata.Rom.TesseractOCRDetectedLangKey] = item.TesseractOCRDetectedLang, + [Models.Metadata.Rom.TesseractOCRDetectedLangConfKey] = item.TesseractOCRDetectedLangConf, + [Models.Metadata.Rom.TesseractOCRDetectedScriptKey] = item.TesseractOCRDetectedScript, + [Models.Metadata.Rom.TesseractOCRDetectedScriptConfKey] = item.TesseractOCRDetectedScriptConf, + [Models.Metadata.Rom.TesseractOCRModuleVersionKey] = item.TesseractOCRModuleVersion, + [Models.Metadata.Rom.TesseractOCRParametersKey] = item.TesseractOCRParameters, + [Models.Metadata.Rom.PDFModuleVersionKey] = item.PDFModuleVersion, + [Models.Metadata.Rom.WordConfidenceInterval0To10Key] = item.WordConfidenceInterval0To10, + [Models.Metadata.Rom.WordConfidenceInterval11To20Key] = item.WordConfidenceInterval11To20, + [Models.Metadata.Rom.WordConfidenceInterval21To30Key] = item.WordConfidenceInterval21To30, + [Models.Metadata.Rom.WordConfidenceInterval31To40Key] = item.WordConfidenceInterval31To40, + [Models.Metadata.Rom.WordConfidenceInterval41To50Key] = item.WordConfidenceInterval41To50, + [Models.Metadata.Rom.WordConfidenceInterval51To60Key] = item.WordConfidenceInterval51To60, + [Models.Metadata.Rom.WordConfidenceInterval61To70Key] = item.WordConfidenceInterval61To70, + [Models.Metadata.Rom.WordConfidenceInterval71To80Key] = item.WordConfidenceInterval71To80, + [Models.Metadata.Rom.WordConfidenceInterval81To90Key] = item.WordConfidenceInterval81To90, + [Models.Metadata.Rom.WordConfidenceInterval91To100Key] = item.WordConfidenceInterval91To100, + + [Models.Metadata.Rom.AlbumKey] = item.Album, + [Models.Metadata.Rom.ArtistKey] = item.Artist, + [Models.Metadata.Rom.BitrateKey] = item.Bitrate, + [Models.Metadata.Rom.CreatorKey] = item.Creator, + [Models.Metadata.Rom.HeightKey] = item.Height, + [Models.Metadata.Rom.LengthKey] = item.Length, + [Models.Metadata.Rom.PreviewImageKey] = item.PreviewImage, + [Models.Metadata.Rom.RotationKey] = item.Rotation, + [Models.Metadata.Rom.TitleKey] = item.Title, + [Models.Metadata.Rom.TrackKey] = item.Track, + [Models.Metadata.Rom.WidthKey] = item.Width, + }; + return rom; + } + } +} \ No newline at end of file diff --git a/Files/ArchiveDotOrg.Deserializer.cs b/Files/ArchiveDotOrg.Deserializer.cs new file mode 100644 index 00000000..dffa5637 --- /dev/null +++ b/Files/ArchiveDotOrg.Deserializer.cs @@ -0,0 +1,7 @@ +namespace SabreTools.Serialization.Files +{ + public partial class ArchiveDotOrg : XmlFile + { + // All serialization logic is in the base class + } +} \ No newline at end of file diff --git a/Streams/ArchiveDotOrg.Deserializer.cs b/Streams/ArchiveDotOrg.Deserializer.cs new file mode 100644 index 00000000..6bb9e778 --- /dev/null +++ b/Streams/ArchiveDotOrg.Deserializer.cs @@ -0,0 +1,7 @@ +namespace SabreTools.Serialization.Streams +{ + public partial class ArchiveDotOrg : XmlFile + { + // All serialization logic is in the base class + } +} \ No newline at end of file