diff --git a/CHANGELIST.md b/CHANGELIST.md index e43bfb75..b55f7def 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -35,6 +35,7 @@ - Update RedumpLib to 1.6.5 - Fix package reference layout - Update Nuget packages +- Normalize Shift-JIS characters, when possible ### 3.3.0 (2025-01-03) diff --git a/MPF.Processors.Test/ProcessingToolTests.cs b/MPF.Processors.Test/ProcessingToolTests.cs index 108c6134..d6efb2d4 100644 --- a/MPF.Processors.Test/ProcessingToolTests.cs +++ b/MPF.Processors.Test/ProcessingToolTests.cs @@ -426,6 +426,46 @@ namespace MPF.Processors.Test #endregion + #region NormalizeShiftJIS + + [Fact] + public void NormalizeShiftJIS_Null_Empty() + { + byte[]? contents = null; + string? actual = ProcessingTool.NormalizeShiftJIS(contents); + Assert.NotNull(actual); + Assert.Empty(actual); + } + + [Fact] + public void NormalizeShiftJIS_Empty_Empty() + { + byte[]? contents = []; + string? actual = ProcessingTool.NormalizeShiftJIS(contents); + Assert.NotNull(actual); + Assert.Empty(actual); + } + + [Fact] + public void NormalizeShiftJIS_NoShiftJIS_Valid() + { + string? expected = "ABCDE"; + byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45]; + string? actual = ProcessingTool.NormalizeShiftJIS(contents); + Assert.Equal(expected, actual); + } + + [Fact] + public void NormalizeShiftJIS_ShiftJIS_Valid() + { + string? expected = "ABCDE ひらがな"; + byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45, 0x20, 0x82, 0xD0, 0x82, 0xE7, 0x82, 0xAA, 0x82, 0xC8]; + string? actual = ProcessingTool.NormalizeShiftJIS(contents); + Assert.Equal(expected, actual); + } + + #endregion + #region GetUMDCategory [Theory] diff --git a/MPF.Processors/ProcessingTool.cs b/MPF.Processors/ProcessingTool.cs index 6ac38a26..b768c54e 100644 --- a/MPF.Processors/ProcessingTool.cs +++ b/MPF.Processors/ProcessingTool.cs @@ -22,6 +22,15 @@ namespace MPF.Processors /// public static class ProcessingTool { + #region Constants + + /// + /// Shift-JIS encoding for detection and conversion + /// + private static readonly Encoding ShiftJIS = Encoding.GetEncoding(932); + + #endregion + #region Information Extraction /// @@ -163,14 +172,15 @@ namespace MPF.Processors if (!File.Exists(filename)) return null; + // Read the entire file as bytes + byte[] bytes = File.ReadAllBytes(filename); + // If we're reading as binary if (binary) - { - byte[] bytes = File.ReadAllBytes(filename); return BitConverter.ToString(bytes).Replace("-", string.Empty); - } - return File.ReadAllText(filename); + // If we're reading as text + return NormalizeShiftJIS(bytes); } /// @@ -292,6 +302,66 @@ namespace MPF.Processors return di.Units[0]?.Body?.DiscTypeIdentifier; } + /// + /// Normalize a byte array that may contain Shift-JIS characters + /// + /// String as a byte array to normalize + /// Normalized version of a string + public static string NormalizeShiftJIS(byte[]? contents) + { + // Invalid arrays are passed as-is + if (contents == null || contents.Length == 0) + return string.Empty; + +#if NET462_OR_GREATER || NETCOREAPP + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); +#endif + + // If the line contains Shift-JIS characters + if (BytesContainsShiftJIS(contents)) + return ShiftJIS.GetString(contents); + + return Encoding.UTF8.GetString(contents); + } + + /// + /// Determine if a byte array contains Shift-JIS encoded characters + /// + /// Byte array to check for Shift-JIS encoding + /// True if the byte array contains Shift-JIS characters, false otherwise + /// + internal static bool BytesContainsShiftJIS(byte[] bytes) + { + // Invalid arrays do not count + if (bytes == null || bytes.Length == 0) + return false; + + // Loop through and check each pair of bytes + for (int i = 0; i < bytes.Length - 1; i++) + { + byte first = bytes[i]; + byte second = bytes[i + 1]; + + if ((first >= 0x81 && first <= 0x84) || + (first >= 0x87 && first <= 0x9F)) + { + if (second >= 0x40 && second <= 0x9E) + return true; + else if (second >= 0x9F && second <= 0xFC) + return true; + } + else if (first >= 0xE0 && first <= 0xEF) + { + if (second >= 0x40 && second <= 0x9E) + return true; + else if (second >= 0x9F && second <= 0xFC) + return true; + } + } + + return false; + } + #endregion #region Category Extraction diff --git a/MPF.Processors/Redumper.cs b/MPF.Processors/Redumper.cs index 545a7268..9d18b6d1 100644 --- a/MPF.Processors/Redumper.cs +++ b/MPF.Processors/Redumper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using System.Text.RegularExpressions; using SabreTools.Hashing; using SabreTools.RedumpLib; @@ -116,7 +117,7 @@ namespace MPF.Processors // Attempt to get the error count long scsiErrors = GetSCSIErrorCount($"{basePath}.log"); - info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString());; + info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString()); // Bluray-specific options if (Type == MediaType.BluRay || Type == MediaType.NintendoWiiUOpticalDisc) @@ -376,7 +377,7 @@ namespace MPF.Processors string? ps2Protection = GetPlayStation2Protection($"{basePath}.log"); if (ps2Protection != null) info.CommonDiscInfo!.Comments = $"Protection: {ps2Protection}" + Environment.NewLine; - + break; case RedumpSystem.SonyPlayStation3: @@ -736,14 +737,16 @@ namespace MPF.Processors return null; // Now that we're at the relevant entries, read each line in and concatenate - string? cueString = string.Empty, line = sr.ReadLine()?.Trim(); + var sb = new StringBuilder(); + string? line = sr.ReadLine()?.Trim(); while (!string.IsNullOrEmpty(line)) { - cueString += line + "\n"; + // TODO: Figure out how to use NormalizeShiftJIS here + sb.AppendLine(line); line = sr.ReadLine()?.Trim(); } - return cueString.TrimEnd('\n'); + return sb.ToString().TrimEnd('\n'); } catch {