From da0bf64e94e93f5e74fe9871d8f5bfa39e8c28bd Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 20:55:20 -0700 Subject: [PATCH 01/12] Add log compression option --- MPF.Library/Data/Options.cs | 9 +++++++++ MPF/Windows/OptionsWindow.xaml | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/MPF.Library/Data/Options.cs b/MPF.Library/Data/Options.cs index d4387de5..9279b6c9 100644 --- a/MPF.Library/Data/Options.cs +++ b/MPF.Library/Data/Options.cs @@ -286,6 +286,15 @@ namespace MPF.Data set { _settings["OutputSubmissionJSON"] = value.ToString(); } } + /// + /// Compress output log files to reduce space + /// + public bool CompressLogFiles + { + get { return GetBooleanSetting(_settings, "CompressLogFiles", true); } + set { _settings["CompressLogFiles"] = value.ToString(); } + } + #endregion #region Skip Options diff --git a/MPF/Windows/OptionsWindow.xaml b/MPF/Windows/OptionsWindow.xaml index 8e62e5e6..73f1e77b 100644 --- a/MPF/Windows/OptionsWindow.xaml +++ b/MPF/Windows/OptionsWindow.xaml @@ -109,6 +109,11 @@ IsChecked="{Binding Path=UIOptions.Options.OutputSubmissionJSON}" ToolTip="Enable outputting a compressed JSON version of the submission info" Margin="0,4" /> + + From 01bf3c9efb76b6d15892e4aa34e38c4f446b714b Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:21:50 -0700 Subject: [PATCH 02/12] Add log file compression --- MPF.Library/Data/BaseParameters.cs | 7 +++++ MPF.Library/Data/DumpEnvironment.cs | 49 +++++++++++++++++++++++++++++ MPF.Library/MPF.Library.csproj | 1 + 3 files changed, 57 insertions(+) diff --git a/MPF.Library/Data/BaseParameters.cs b/MPF.Library/Data/BaseParameters.cs index 23436a8b..26d23f85 100644 --- a/MPF.Library/Data/BaseParameters.cs +++ b/MPF.Library/Data/BaseParameters.cs @@ -160,6 +160,13 @@ namespace MPF.Data /// public bool IsValid() => GenerateParameters() != null; + /// + /// Generate a list of all log files generated + /// + /// Base filename and path to use for checking + /// List of all log file paths, empty otherwise + public List GetLogFilePaths(string basePath) => new List(); + /// /// Reset all special variables to have default values /// diff --git a/MPF.Library/Data/DumpEnvironment.cs b/MPF.Library/Data/DumpEnvironment.cs index 8408c479..f45f68ef 100644 --- a/MPF.Library/Data/DumpEnvironment.cs +++ b/MPF.Library/Data/DumpEnvironment.cs @@ -478,6 +478,10 @@ namespace MPF.Data resultProgress?.Report(Result.Failure("Writing could not complete!")); } + // Conpress the logs, if required + if (Options.CompressLogFiles) + CompressLogFiles(); + resultProgress?.Report(Result.Success("Submission information process complete!")); return Result.Success(); } @@ -1292,6 +1296,51 @@ namespace MPF.Data return true; } + /// + /// Compress log files to save space + /// + /// True if the process succeeded, false otherwise + private bool CompressLogFiles() + { + // Prepare the necessary paths + string outputFilename = Path.GetFileNameWithoutExtension(OutputFilename); + string combinedBase = Path.Combine(OutputDirectory, outputFilename); + string archiveName = Path.Combine(OutputDirectory, "!dumpinfo.zip"); + + // Get the list of log files from the parameters object + var files = Parameters.GetLogFilePaths(combinedBase); + if (!files.Any()) + return true; + + // Add the log files to the archive and delete the uncompressed file after + ZipArchive zf = null; + try + { + zf = ZipFile.Open(archiveName, ZipArchiveMode.Create); + foreach (string file in files) + { + string entryName = file.Substring(OutputDirectory.Length).TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + zf.CreateEntryFromFile(file, entryName); + + try + { + File.Delete(file); + } + catch { } + } + + return true; + } + catch + { + return false; + } + finally + { + zf?.Dispose(); + } + } + #endregion #region Information Extraction diff --git a/MPF.Library/MPF.Library.csproj b/MPF.Library/MPF.Library.csproj index b70fa54b..b7904a7a 100644 --- a/MPF.Library/MPF.Library.csproj +++ b/MPF.Library/MPF.Library.csproj @@ -71,6 +71,7 @@ runtime; compile; build; native; analyzers; buildtransitive + From 96fa8d8cef4374a791134198d6a1ec3ad47ada56 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:36:14 -0700 Subject: [PATCH 03/12] Add DiscImageCreator log paths --- MPF.Library/Data/BaseParameters.cs | 14 +-- MPF.Library/DiscImageCreator/Parameters.cs | 104 +++++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/MPF.Library/Data/BaseParameters.cs b/MPF.Library/Data/BaseParameters.cs index 26d23f85..51b74b3c 100644 --- a/MPF.Library/Data/BaseParameters.cs +++ b/MPF.Library/Data/BaseParameters.cs @@ -142,6 +142,13 @@ namespace MPF.Data /// String representing the media type, null on error public virtual string GetDefaultExtension(MediaType? mediaType) => null; + /// + /// Generate a list of all log files generated + /// + /// Base filename and path to use for checking + /// List of all log file paths, empty otherwise + public virtual List GetLogFilePaths(string basePath) => new List(); + /// /// Get the MediaType from the current set of parameters /// @@ -160,13 +167,6 @@ namespace MPF.Data /// public bool IsValid() => GenerateParameters() != null; - /// - /// Generate a list of all log files generated - /// - /// Base filename and path to use for checking - /// List of all log file paths, empty otherwise - public List GetLogFilePaths(string basePath) => new List(); - /// /// Reset all special variables to have default values /// diff --git a/MPF.Library/DiscImageCreator/Parameters.cs b/MPF.Library/DiscImageCreator/Parameters.cs index ffdd3ccd..8b56012d 100644 --- a/MPF.Library/DiscImageCreator/Parameters.cs +++ b/MPF.Library/DiscImageCreator/Parameters.cs @@ -1216,6 +1216,110 @@ namespace MPF.DiscImageCreator /// public override string GetDefaultExtension(MediaType? mediaType) => Converters.Extension(mediaType); + /// + public override List GetLogFilePaths(string basePath) + { + var currentFiles = Directory.GetFiles(Path.GetDirectoryName(basePath)); + string cmdPath = currentFiles.FirstOrDefault(f => Regex.IsMatch(f, @"\d{8}T\d{6}")); + + // TODO: Figure out how to get the filename for the former _cmd file... + List logFiles = new List(); + switch (this.Type) + { + case MediaType.CDROM: + case MediaType.GDROM: // TODO: Verify GD-ROM outputs this + if (File.Exists($"{basePath}.c2")) + logFiles.Add($"{basePath}.c2"); + if (File.Exists($"{basePath}_c2Error.txt")) + logFiles.Add($"{basePath}_c2Error.txt"); + if (File.Exists($"{basePath}.ccd")) + logFiles.Add($"{basePath}.ccd"); + if (cmdPath != null && File.Exists(cmdPath)) + logFiles.Add(cmdPath); + if (File.Exists($"{basePath}_cmd.txt")) + logFiles.Add($"{basePath}_cmd.txt"); + if (File.Exists($"{basePath}.dat")) + logFiles.Add($"{basePath}.dat"); + if (File.Exists($"{basePath}.sub")) + logFiles.Add($"{basePath}.sub"); + if (File.Exists($"{basePath}.subtmp")) + logFiles.Add($"{basePath}.subtmp"); + if (File.Exists($"{basePath}_disc.txt")) + logFiles.Add($"{basePath}_disc.txt"); + if (File.Exists($"{basePath}_drive.txt")) + logFiles.Add($"{basePath}_drive.txt"); + if (File.Exists($"{basePath}_img.cue")) + logFiles.Add($"{basePath}_img.cue"); + if (File.Exists($"{basePath}_mainError.txt")) + logFiles.Add($"{basePath}_mainError.txt"); + if (File.Exists($"{basePath}_mainInfo.txt")) + logFiles.Add($"{basePath}_mainInfo.txt"); + if (File.Exists($"{basePath}_sub.txt")) + logFiles.Add($"{basePath}_sub.txt"); + if (File.Exists($"{basePath}_subError.txt")) + logFiles.Add($"{basePath}_subError.txt"); + if (File.Exists($"{basePath}_subInfo.txt")) + logFiles.Add($"{basePath}_subInfo.txt"); + if (File.Exists($"{basePath}_subIntention.txt")) + logFiles.Add($"{basePath}_subIntention.txt"); + if (File.Exists($"{basePath}_subReadable.txt")) + logFiles.Add($"{basePath}_subReadable.txt"); + if (File.Exists($"{basePath}_volDesc.txt")) + logFiles.Add($"{basePath}_volDesc.txt"); + + // Audio-only discs don't output these files + if (!this.System.IsAudio()) + { + if (File.Exists($"{basePath}.img_EdcEcc.txt")) + logFiles.Add($"{basePath}.img_EdcEcc.txt"); + if (File.Exists($"{basePath}.img_EccEdc.txt")) + logFiles.Add($"{basePath}.img_EccEdc.txt"); + } + + break; + + case MediaType.DVD: + case MediaType.HDDVD: + case MediaType.BluRay: + case MediaType.NintendoGameCubeGameDisc: + case MediaType.NintendoWiiOpticalDisc: + if (cmdPath != null && File.Exists(cmdPath)) + logFiles.Add(cmdPath); + if (File.Exists($"{basePath}_cmd.txt")) + logFiles.Add($"{basePath}_cmd.txt"); + if (File.Exists($"{basePath}.dat")) + logFiles.Add($"{basePath}.dat"); + if (File.Exists($"{basePath}_disc.txt")) + logFiles.Add($"{basePath}_disc.txt"); + if (File.Exists($"{basePath}_drive.txt")) + logFiles.Add($"{basePath}_drive.txt"); + if (File.Exists($"{basePath}_mainError.txt")) + logFiles.Add($"{basePath}_mainError.txt"); + if (File.Exists($"{basePath}_mainInfo.txt")) + logFiles.Add($"{basePath}_mainInfo.txt"); + if (File.Exists($"{basePath}_volDesc.txt")) + logFiles.Add($"{basePath}_volDesc.txt"); + + break; + + case MediaType.FloppyDisk: + case MediaType.HardDisk: + // TODO: Determine what outputs come out from a HDD, SD, etc. + if (cmdPath != null && File.Exists(cmdPath)) + logFiles.Add(cmdPath); + if (File.Exists($"{basePath}_cmd.txt")) + logFiles.Add($"{basePath}_cmd.txt"); + if (File.Exists($"{basePath}.dat")) + logFiles.Add($"{basePath}.dat"); + if (File.Exists($"{basePath}_disc.txt")) + logFiles.Add($"{basePath}_disc.txt"); + + break; + } + + return logFiles; + } + /// public override MediaType? GetMediaType() => Converters.ToMediaType(BaseCommand); From 3230d59f6abac163092dd6faa22494fb3d4ab4fb Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:49:57 -0700 Subject: [PATCH 04/12] Fix options window --- MPF/Windows/OptionsWindow.xaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MPF/Windows/OptionsWindow.xaml b/MPF/Windows/OptionsWindow.xaml index 73f1e77b..12950004 100644 --- a/MPF/Windows/OptionsWindow.xaml +++ b/MPF/Windows/OptionsWindow.xaml @@ -112,7 +112,7 @@ From ad5cd5b8f9ba6c12e1ab0641b80a1379f8c775ca Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:53:41 -0700 Subject: [PATCH 05/12] Only encode artifacts if we're outputting JSON --- MPF.Library/Aaru/Parameters.cs | 31 ++++--- MPF.Library/CleanRIp/Parameters.cs | 15 ++-- MPF.Library/DD/Parameters.cs | 2 +- MPF.Library/Data/BaseParameters.cs | 3 +- MPF.Library/Data/DumpEnvironment.cs | 2 +- MPF.Library/DiscImageCreator/Parameters.cs | 99 +++++++++++----------- MPF.Library/UmdImageCreator/Parameters.cs | 23 ++--- 7 files changed, 94 insertions(+), 81 deletions(-) diff --git a/MPF.Library/Aaru/Parameters.cs b/MPF.Library/Aaru/Parameters.cs index e400061d..e3f852de 100644 --- a/MPF.Library/Aaru/Parameters.cs +++ b/MPF.Library/Aaru/Parameters.cs @@ -205,7 +205,7 @@ namespace MPF.Aaru } /// - public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive) + public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive, bool includeArtifacts) { // TODO: Fill in submission info specifics for Aaru string outputDirectory = Path.GetDirectoryName(basePath); @@ -365,19 +365,22 @@ namespace MPF.Aaru break; } - // Fill in any artifacts that exist, Base64-encoded - if (File.Exists(basePath + ".cicm.xml")) - info.Artifacts["cicm"] = GetBase64(GetFullFile(basePath + ".cicm.xml")); - if (File.Exists(basePath + ".ibg")) - info.Artifacts["ibg"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".ibg")); - if (File.Exists(basePath + ".log")) - info.Artifacts["log"] = GetBase64(GetFullFile(basePath + ".log")); - if (File.Exists(basePath + ".mhddlog.bin")) - info.Artifacts["mhddlog_bin"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".mhddlog.bin")); - if (File.Exists(basePath + ".resume.xml")) - info.Artifacts["resume"] = GetBase64(GetFullFile(basePath + ".resume.xml")); - if (File.Exists(basePath + ".sub.log")) - info.Artifacts["sub_log"] = GetBase64(GetFullFile(basePath + ".sub.log")); + // Fill in any artifacts that exist, Base64-encoded, if we need to + if (includeArtifacts) + { + if (File.Exists(basePath + ".cicm.xml")) + info.Artifacts["cicm"] = GetBase64(GetFullFile(basePath + ".cicm.xml")); + if (File.Exists(basePath + ".ibg")) + info.Artifacts["ibg"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".ibg")); + if (File.Exists(basePath + ".log")) + info.Artifacts["log"] = GetBase64(GetFullFile(basePath + ".log")); + if (File.Exists(basePath + ".mhddlog.bin")) + info.Artifacts["mhddlog_bin"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".mhddlog.bin")); + if (File.Exists(basePath + ".resume.xml")) + info.Artifacts["resume"] = GetBase64(GetFullFile(basePath + ".resume.xml")); + if (File.Exists(basePath + ".sub.log")) + info.Artifacts["sub_log"] = GetBase64(GetFullFile(basePath + ".sub.log")); + } } /// diff --git a/MPF.Library/CleanRIp/Parameters.cs b/MPF.Library/CleanRIp/Parameters.cs index 3a57a5f5..c9ce3006 100644 --- a/MPF.Library/CleanRIp/Parameters.cs +++ b/MPF.Library/CleanRIp/Parameters.cs @@ -54,7 +54,7 @@ namespace MPF.CleanRip } /// - public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive) + public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive, bool includeArtifacts) { info.TracksAndWriteOffsets.ClrMameProData = GetCleanripDatfile(basePath + ".iso", basePath + "-dumpinfo.txt"); @@ -89,11 +89,14 @@ namespace MPF.CleanRip break; } - // Fill in any artifacts that exist, Base64-encoded - if (File.Exists(basePath + ".bca")) - info.Artifacts["bca"] = GetBase64(GetFullFile(basePath + ".bca", binary: true)); - if (File.Exists(basePath + "-dumpinfo.txt")) - info.Artifacts["dumpinfo"] = GetBase64(GetFullFile(basePath + "-dumpinfo.txt")); + // Fill in any artifacts that exist, Base64-encoded, if we need to + if (includeArtifacts) + { + if (File.Exists(basePath + ".bca")) + info.Artifacts["bca"] = GetBase64(GetFullFile(basePath + ".bca", binary: true)); + if (File.Exists(basePath + "-dumpinfo.txt")) + info.Artifacts["dumpinfo"] = GetBase64(GetFullFile(basePath + "-dumpinfo.txt")); + } } #endregion diff --git a/MPF.Library/DD/Parameters.cs b/MPF.Library/DD/Parameters.cs index 5eff05b0..bf102d11 100644 --- a/MPF.Library/DD/Parameters.cs +++ b/MPF.Library/DD/Parameters.cs @@ -101,7 +101,7 @@ namespace MPF.DD } /// - public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive) + public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive, bool includeArtifacts) { // TODO: Fill in submission info specifics for DD string outputDirectory = Path.GetDirectoryName(basePath); diff --git a/MPF.Library/Data/BaseParameters.cs b/MPF.Library/Data/BaseParameters.cs index 51b74b3c..50942936 100644 --- a/MPF.Library/Data/BaseParameters.cs +++ b/MPF.Library/Data/BaseParameters.cs @@ -123,7 +123,8 @@ namespace MPF.Data /// Base submission info to fill in specifics for /// Base filename and path to use for checking /// Drive representing the disc to get information from - public abstract void GenerateSubmissionInfo(SubmissionInfo submissionInfo, string basePath, Drive drive); + /// True to include output files as encoded artifacts, false otherwise + public abstract void GenerateSubmissionInfo(SubmissionInfo submissionInfo, string basePath, Drive drive, bool includeArtifacts); #endregion diff --git a/MPF.Library/Data/DumpEnvironment.cs b/MPF.Library/Data/DumpEnvironment.cs index f45f68ef..8d34f22d 100644 --- a/MPF.Library/Data/DumpEnvironment.cs +++ b/MPF.Library/Data/DumpEnvironment.cs @@ -601,7 +601,7 @@ namespace MPF.Data }; // Get specific tool output handling - Parameters.GenerateSubmissionInfo(info, combinedBase, this.Drive); + Parameters.GenerateSubmissionInfo(info, combinedBase, this.Drive, Options.OutputSubmissionJSON); // Get a list of matching IDs for each line in the DAT if (!string.IsNullOrEmpty(info.TracksAndWriteOffsets.ClrMameProData) && Options.HasRedumpLogin) diff --git a/MPF.Library/DiscImageCreator/Parameters.cs b/MPF.Library/DiscImageCreator/Parameters.cs index 8b56012d..7f5f9bcb 100644 --- a/MPF.Library/DiscImageCreator/Parameters.cs +++ b/MPF.Library/DiscImageCreator/Parameters.cs @@ -354,7 +354,7 @@ namespace MPF.DiscImageCreator } /// - public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive) + public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive, bool includeArtifacts) { string outputDirectory = Path.GetDirectoryName(basePath); @@ -708,53 +708,56 @@ namespace MPF.DiscImageCreator break; } - // Fill in any artifacts that exist, Base64-encoded - //if (File.Exists(basePath + ".c2")) - // info.Artifacts["c2"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".c2")); - if (File.Exists(basePath + "_c2Error.txt")) - info.Artifacts["c2Error"] = GetBase64(GetFullFile(basePath + "_c2Error.txt")); - if (File.Exists(basePath + ".ccd")) - info.Artifacts["ccd"] = GetBase64(GetFullFile(basePath + ".ccd")); - if (File.Exists(basePath + "_cmd.txt")) // TODO: Figure out how to read in the timestamp-named file - info.Artifacts["cmd"] = GetBase64(GetFullFile(basePath + "_cmd.txt")); - if (File.Exists(basePath + ".cue")) - info.Artifacts["cue"] = GetBase64(GetFullFile(basePath + ".cue")); - if (File.Exists(basePath + ".dat")) - info.Artifacts["dat"] = GetBase64(GetFullFile(basePath + ".dat")); - if (File.Exists(basePath + "_disc.txt")) - info.Artifacts["disc"] = GetBase64(GetFullFile(basePath + "_disc.txt")); - //if (File.Exists(Path.Combine(outputDirectory, "DMI.bin"))) - // info.Artifacts["dmi"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "DMI.bin"))); - if (File.Exists(basePath + "_drive.txt")) - info.Artifacts["drive"] = GetBase64(GetFullFile(basePath + "_drive.txt")); - if (File.Exists(basePath + "_img.cue")) - info.Artifacts["img_cue"] = GetBase64(GetFullFile(basePath + "_img.cue")); - if (File.Exists(basePath + ".img_EdcEcc.txt")) - info.Artifacts["img_EdcEcc"] = GetBase64(GetFullFile(basePath + ".img_EdcEcc.txt")); - if (File.Exists(basePath + ".img_EccEdc.txt")) - info.Artifacts["img_EdcEcc"] = GetBase64(GetFullFile(basePath + ".img_EccEdc.txt")); - if (File.Exists(basePath + "_mainError.txt")) - info.Artifacts["mainError"] = GetBase64(GetFullFile(basePath + "_mainError.txt")); - if (File.Exists(basePath + "_mainInfo.txt")) - info.Artifacts["mainInfo"] = GetBase64(GetFullFile(basePath + "_mainInfo.txt")); - //if (File.Exists(Path.Combine(outputDirectory, "PFI.bin"))) - // info.Artifacts["pfi"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "PFI.bin"))); - //if (File.Exists(Path.Combine(outputDirectory, "SS.bin"))) - // info.Artifacts["ss"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "SS.bin"))); - if (File.Exists(basePath + ".sub")) - info.Artifacts["sub"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".sub")); - if (File.Exists(basePath + "_subError.txt")) - info.Artifacts["subError"] = GetBase64(GetFullFile(basePath + "_subError.txt")); - if (File.Exists(basePath + "_subInfo.txt")) - info.Artifacts["subInfo"] = GetBase64(GetFullFile(basePath + "_subInfo.txt")); - if (File.Exists(basePath + "_subIntention.txt")) - info.Artifacts["subIntention"] = GetBase64(GetFullFile(basePath + "_subIntention.txt")); - //if (File.Exists(basePath + "_sub.txt")) - // info.Artifacts["subReadable"] = GetBase64(GetFullFile(basePath + "_sub.txt")); - //if (File.Exists(basePath + "_subReadable.txt")) - // info.Artifacts["subReadable"] = GetBase64(GetFullFile(basePath + "_subReadable.txt")); - if (File.Exists(basePath + "_volDesc.txt")) - info.Artifacts["volDesc"] = GetBase64(GetFullFile(basePath + "_volDesc.txt")); + // Fill in any artifacts that exist, Base64-encoded, if we need to + if (includeArtifacts) + { + //if (File.Exists(basePath + ".c2")) + // info.Artifacts["c2"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".c2")); + if (File.Exists(basePath + "_c2Error.txt")) + info.Artifacts["c2Error"] = GetBase64(GetFullFile(basePath + "_c2Error.txt")); + if (File.Exists(basePath + ".ccd")) + info.Artifacts["ccd"] = GetBase64(GetFullFile(basePath + ".ccd")); + if (File.Exists(basePath + "_cmd.txt")) // TODO: Figure out how to read in the timestamp-named file + info.Artifacts["cmd"] = GetBase64(GetFullFile(basePath + "_cmd.txt")); + if (File.Exists(basePath + ".cue")) + info.Artifacts["cue"] = GetBase64(GetFullFile(basePath + ".cue")); + if (File.Exists(basePath + ".dat")) + info.Artifacts["dat"] = GetBase64(GetFullFile(basePath + ".dat")); + if (File.Exists(basePath + "_disc.txt")) + info.Artifacts["disc"] = GetBase64(GetFullFile(basePath + "_disc.txt")); + //if (File.Exists(Path.Combine(outputDirectory, "DMI.bin"))) + // info.Artifacts["dmi"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "DMI.bin"))); + if (File.Exists(basePath + "_drive.txt")) + info.Artifacts["drive"] = GetBase64(GetFullFile(basePath + "_drive.txt")); + if (File.Exists(basePath + "_img.cue")) + info.Artifacts["img_cue"] = GetBase64(GetFullFile(basePath + "_img.cue")); + if (File.Exists(basePath + ".img_EdcEcc.txt")) + info.Artifacts["img_EdcEcc"] = GetBase64(GetFullFile(basePath + ".img_EdcEcc.txt")); + if (File.Exists(basePath + ".img_EccEdc.txt")) + info.Artifacts["img_EdcEcc"] = GetBase64(GetFullFile(basePath + ".img_EccEdc.txt")); + if (File.Exists(basePath + "_mainError.txt")) + info.Artifacts["mainError"] = GetBase64(GetFullFile(basePath + "_mainError.txt")); + if (File.Exists(basePath + "_mainInfo.txt")) + info.Artifacts["mainInfo"] = GetBase64(GetFullFile(basePath + "_mainInfo.txt")); + //if (File.Exists(Path.Combine(outputDirectory, "PFI.bin"))) + // info.Artifacts["pfi"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "PFI.bin"))); + //if (File.Exists(Path.Combine(outputDirectory, "SS.bin"))) + // info.Artifacts["ss"] = Convert.ToBase64String(File.ReadAllBytes(Path.Combine(outputDirectory, "SS.bin"))); + if (File.Exists(basePath + ".sub")) + info.Artifacts["sub"] = Convert.ToBase64String(File.ReadAllBytes(basePath + ".sub")); + if (File.Exists(basePath + "_subError.txt")) + info.Artifacts["subError"] = GetBase64(GetFullFile(basePath + "_subError.txt")); + if (File.Exists(basePath + "_subInfo.txt")) + info.Artifacts["subInfo"] = GetBase64(GetFullFile(basePath + "_subInfo.txt")); + if (File.Exists(basePath + "_subIntention.txt")) + info.Artifacts["subIntention"] = GetBase64(GetFullFile(basePath + "_subIntention.txt")); + //if (File.Exists(basePath + "_sub.txt")) + // info.Artifacts["subReadable"] = GetBase64(GetFullFile(basePath + "_sub.txt")); + //if (File.Exists(basePath + "_subReadable.txt")) + // info.Artifacts["subReadable"] = GetBase64(GetFullFile(basePath + "_subReadable.txt")); + if (File.Exists(basePath + "_volDesc.txt")) + info.Artifacts["volDesc"] = GetBase64(GetFullFile(basePath + "_volDesc.txt")); + } } /// diff --git a/MPF.Library/UmdImageCreator/Parameters.cs b/MPF.Library/UmdImageCreator/Parameters.cs index bfc15154..8d1fd916 100644 --- a/MPF.Library/UmdImageCreator/Parameters.cs +++ b/MPF.Library/UmdImageCreator/Parameters.cs @@ -56,7 +56,7 @@ namespace MPF.UmdImageCreator } /// - public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive) + public override void GenerateSubmissionInfo(SubmissionInfo info, string basePath, Drive drive, bool includeArtifacts) { // Extract info based generically on MediaType switch (this.Type) @@ -86,15 +86,18 @@ namespace MPF.UmdImageCreator break; } - // Fill in any artifacts that exist, Base64-encoded - if (File.Exists(basePath + "_disc.txt")) - info.Artifacts["disc"] = GetBase64(GetFullFile(basePath + "_disc.txt")); - if (File.Exists(basePath + "_mainError.txt")) - info.Artifacts["mainError"] = GetBase64(GetFullFile(basePath + "_mainError.txt")); - if (File.Exists(basePath + "_mainInfo.txt")) - info.Artifacts["mainInfo"] = GetBase64(GetFullFile(basePath + "_mainInfo.txt")); - if (File.Exists(basePath + "_volDesc.txt")) - info.Artifacts["volDesc"] = GetBase64(GetFullFile(basePath + "_volDesc.txt")); + // Fill in any artifacts that exist, Base64-encoded, if we need to + if (includeArtifacts) + { + if (File.Exists(basePath + "_disc.txt")) + info.Artifacts["disc"] = GetBase64(GetFullFile(basePath + "_disc.txt")); + if (File.Exists(basePath + "_mainError.txt")) + info.Artifacts["mainError"] = GetBase64(GetFullFile(basePath + "_mainError.txt")); + if (File.Exists(basePath + "_mainInfo.txt")) + info.Artifacts["mainInfo"] = GetBase64(GetFullFile(basePath + "_mainInfo.txt")); + if (File.Exists(basePath + "_volDesc.txt")) + info.Artifacts["volDesc"] = GetBase64(GetFullFile(basePath + "_volDesc.txt")); + } } #endregion From 096a8a6a06dac9cf77b1377de09663a489252bf8 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:56:14 -0700 Subject: [PATCH 06/12] Add Aaru log file paths, remove TODO in DIC --- MPF.Library/Aaru/Parameters.cs | 42 ++++++++++++++++++++++ MPF.Library/DiscImageCreator/Parameters.cs | 1 - 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/MPF.Library/Aaru/Parameters.cs b/MPF.Library/Aaru/Parameters.cs index e3f852de..aeb6887c 100644 --- a/MPF.Library/Aaru/Parameters.cs +++ b/MPF.Library/Aaru/Parameters.cs @@ -1033,6 +1033,48 @@ namespace MPF.Aaru /// public override string GetDefaultExtension(MediaType? mediaType) => Converters.Extension(mediaType); + /// + public override List GetLogFilePaths(string basePath) + { + List logFiles = new List(); + switch (this.Type) + { + case MediaType.CDROM: + if (File.Exists($"{basePath}.cicm.xml")) + logFiles.Add($"{basePath}.cicm.xml"); + if (File.Exists($"{basePath}.ibg")) + logFiles.Add($"{basePath}.ibg"); + if (File.Exists($"{basePath}.log")) + logFiles.Add($"{basePath}.log"); + if (File.Exists($"{basePath}.mhddlog.bin")) + logFiles.Add($"{basePath}.mhddlog.bin"); + if (File.Exists($"{basePath}.resume.xml")) + logFiles.Add($"{basePath}.resume.xml"); + if (File.Exists($"{basePath}.sub.log")) + logFiles.Add($"{basePath}.sub.log"); + + break; + + case MediaType.DVD: + case MediaType.HDDVD: + case MediaType.BluRay: + if (File.Exists($"{basePath}.cicm.xml")) + logFiles.Add($"{basePath}.cicm.xml"); + if (File.Exists($"{basePath}.ibg")) + logFiles.Add($"{basePath}.ibg"); + if (File.Exists($"{basePath}.log")) + logFiles.Add($"{basePath}.log"); + if (File.Exists($"{basePath}.mhddlog.bin")) + logFiles.Add($"{basePath}.mhddlog.bin"); + if (File.Exists($"{basePath}.resume.xml")) + logFiles.Add($"{basePath}.resume.xml"); + + break; + } + + return logFiles; + } + /// public override bool IsDumpingCommand() { diff --git a/MPF.Library/DiscImageCreator/Parameters.cs b/MPF.Library/DiscImageCreator/Parameters.cs index 7f5f9bcb..09e12de7 100644 --- a/MPF.Library/DiscImageCreator/Parameters.cs +++ b/MPF.Library/DiscImageCreator/Parameters.cs @@ -1225,7 +1225,6 @@ namespace MPF.DiscImageCreator var currentFiles = Directory.GetFiles(Path.GetDirectoryName(basePath)); string cmdPath = currentFiles.FirstOrDefault(f => Regex.IsMatch(f, @"\d{8}T\d{6}")); - // TODO: Figure out how to get the filename for the former _cmd file... List logFiles = new List(); switch (this.Type) { From 4fd51dbe4528bb3f5b423277cac7fd1a9c4b73d2 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 21:57:00 -0700 Subject: [PATCH 07/12] Default to false, for now --- MPF.Library/Data/Options.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MPF.Library/Data/Options.cs b/MPF.Library/Data/Options.cs index 9279b6c9..7934cb2f 100644 --- a/MPF.Library/Data/Options.cs +++ b/MPF.Library/Data/Options.cs @@ -291,7 +291,7 @@ namespace MPF.Data /// public bool CompressLogFiles { - get { return GetBooleanSetting(_settings, "CompressLogFiles", true); } + get { return GetBooleanSetting(_settings, "CompressLogFiles", false); } set { _settings["CompressLogFiles"] = value.ToString(); } } From 742db4c8546bd158e2c92a6131104e8e709be25d Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 22:12:32 -0700 Subject: [PATCH 08/12] Change default archive name --- MPF.Library/Data/DumpEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MPF.Library/Data/DumpEnvironment.cs b/MPF.Library/Data/DumpEnvironment.cs index 8d34f22d..78813539 100644 --- a/MPF.Library/Data/DumpEnvironment.cs +++ b/MPF.Library/Data/DumpEnvironment.cs @@ -1305,7 +1305,7 @@ namespace MPF.Data // Prepare the necessary paths string outputFilename = Path.GetFileNameWithoutExtension(OutputFilename); string combinedBase = Path.Combine(OutputDirectory, outputFilename); - string archiveName = Path.Combine(OutputDirectory, "!dumpinfo.zip"); + string archiveName = combinedBase + "_logs.zip"; // Get the list of log files from the parameters object var files = Parameters.GetLogFilePaths(combinedBase); From 1a52a3a205c84e12904a35c4eb39035af1237897 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 22:20:15 -0700 Subject: [PATCH 09/12] Add Check parameter for compression --- MPF.Check/Program.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/MPF.Check/Program.cs b/MPF.Check/Program.cs index 061715cd..4d5f4f77 100644 --- a/MPF.Check/Program.cs +++ b/MPF.Check/Program.cs @@ -65,7 +65,7 @@ namespace MPF.Check string username = null, password = null; string internalProgram = "DiscImageCreator"; string path = string.Empty; - bool scan = false; + bool scan = false, compress = false; // Loop through and process options int startIndex = 2; @@ -113,6 +113,12 @@ namespace MPF.Check scan = true; } + // Compress log and extraneous files + else if (args[startIndex].StartsWith("-z") || args[startIndex].StartsWith("--zip")) + { + compress = true; + } + // Default, we fall out else { @@ -160,6 +166,7 @@ namespace MPF.Check InternalProgram = Converters.ToInternalProgram(internalProgram), ScanForProtection = scan && !string.IsNullOrWhiteSpace(path), PromptForDiscInformation = false, + CompressLogFiles = compress, RedumpUsername = username, RedumpPassword = password, @@ -200,6 +207,7 @@ namespace MPF.Check Console.WriteLine("-u, --use Dumping program output type"); Console.WriteLine("-p, --path Physical drive path for additional checks"); Console.WriteLine("-s, --scan Enable copy protection scan (requires --path)"); + Console.WriteLine("-z, --zip Enable log file compression"); Console.WriteLine(); } From 7e5952bbb8ec557d3a7cd28565d21084439075b4 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 22:23:15 -0700 Subject: [PATCH 10/12] Add log file lsits for CleanRip and UIC --- MPF.Library/CleanRIp/Parameters.cs | 20 ++++++++++++++++++++ MPF.Library/UmdImageCreator/Parameters.cs | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/MPF.Library/CleanRIp/Parameters.cs b/MPF.Library/CleanRIp/Parameters.cs index c9ce3006..84008c0c 100644 --- a/MPF.Library/CleanRIp/Parameters.cs +++ b/MPF.Library/CleanRIp/Parameters.cs @@ -99,6 +99,26 @@ namespace MPF.CleanRip } } + /// + public override List GetLogFilePaths(string basePath) + { + List logFiles = new List(); + switch (this.Type) + { + case MediaType.DVD: // Only added here to help users; not strictly correct + case MediaType.NintendoGameCubeGameDisc: + case MediaType.NintendoWiiOpticalDisc: + if (File.Exists($"{basePath}-dumpinfo.txt")) + logFiles.Add($"{basePath}-dumpinfo.txt"); + if (File.Exists($"{basePath}.bca")) + logFiles.Add($"{basePath}.bca"); + + break; + } + + return logFiles; + } + #endregion #region Information Extraction Methods diff --git a/MPF.Library/UmdImageCreator/Parameters.cs b/MPF.Library/UmdImageCreator/Parameters.cs index 8d1fd916..5682d259 100644 --- a/MPF.Library/UmdImageCreator/Parameters.cs +++ b/MPF.Library/UmdImageCreator/Parameters.cs @@ -100,6 +100,28 @@ namespace MPF.UmdImageCreator } } + /// + public override List GetLogFilePaths(string basePath) + { + List logFiles = new List(); + switch (this.Type) + { + case MediaType.UMD: + if (File.Exists($"{basePath}_disc.txt")) + logFiles.Add($"{basePath}_disc.txt"); + if (File.Exists($"{basePath}_mainError.txt")) + logFiles.Add($"{basePath}_mainError.txt"); + if (File.Exists($"{basePath}_mainInfo.txt")) + logFiles.Add($"{basePath}_mainInfo.txt"); + if (File.Exists($"{basePath}_volDesc.txt")) + logFiles.Add($"{basePath}_volDesc.txt"); + + break; + } + + return logFiles; + } + #endregion #region Information Extraction Methods From 87199c1b0d2ef9bf892754875649aa34291e1a2d Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 22:28:01 -0700 Subject: [PATCH 11/12] Update changelog --- CHANGELIST.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELIST.md b/CHANGELIST.md index 933b60e4..65cebd07 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -45,6 +45,7 @@ - Implement internal log queue - It's a secret... - Updated to DIC version 20210401 +- Support log file compression ### 1.18 (2020-11-10) - Add more information extraction and generation for Aaru From 22b15710d0042089862c5250050ce99521100578 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 2 Apr 2021 22:34:46 -0700 Subject: [PATCH 12/12] Take output of compression into account for log --- MPF.Library/Data/DumpEnvironment.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/MPF.Library/Data/DumpEnvironment.cs b/MPF.Library/Data/DumpEnvironment.cs index 78813539..da67b98f 100644 --- a/MPF.Library/Data/DumpEnvironment.cs +++ b/MPF.Library/Data/DumpEnvironment.cs @@ -480,7 +480,14 @@ namespace MPF.Data // Conpress the logs, if required if (Options.CompressLogFiles) - CompressLogFiles(); + { + resultProgress?.Report(Result.Success("Compressing log files...")); + success = CompressLogFiles(); + if (success) + resultProgress?.Report(Result.Success("Compression complete!")); + else + resultProgress?.Report(Result.Failure("Compression could not complete!")); + } resultProgress?.Report(Result.Success("Submission information process complete!")); return Result.Success();