Normalize Shift-JIS characters, when possible (fixes #827)

CHANGELIST.md

@@ -35,6 +35,7 @@
 - Update RedumpLib to 1.6.5
 - Fix package reference layout
 - Update Nuget packages
+- Normalize Shift-JIS characters, when possible
 
 ### 3.3.0 (2025-01-03)
 

MPF.Processors.Test/ProcessingToolTests.cs

@@ -426,6 +426,46 @@ namespace MPF.Processors.Test
 
         #endregion
 
+        #region NormalizeShiftJIS
+
+        [Fact]
+        public void NormalizeShiftJIS_Null_Empty()
+        {
+            byte[]? contents = null;
+            string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+            Assert.NotNull(actual);
+            Assert.Empty(actual);
+        }
+
+        [Fact]
+        public void NormalizeShiftJIS_Empty_Empty()
+        {
+            byte[]? contents = [];
+            string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+            Assert.NotNull(actual);
+            Assert.Empty(actual);
+        }
+
+        [Fact]
+        public void NormalizeShiftJIS_NoShiftJIS_Valid()
+        {
+            string? expected = "ABCDE";
+            byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45];
+            string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+            Assert.Equal(expected, actual);
+        }
+
+        [Fact]
+        public void NormalizeShiftJIS_ShiftJIS_Valid()
+        {
+            string? expected = "ABCDE ひらがな";
+            byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45, 0x20, 0x82, 0xD0, 0x82, 0xE7, 0x82, 0xAA, 0x82, 0xC8];
+            string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+            Assert.Equal(expected, actual);
+        }
+
+        #endregion
+
         #region GetUMDCategory
 
         [Theory]

MPF.Processors/ProcessingTool.cs

@@ -22,6 +22,15 @@ namespace MPF.Processors
     /// </summary>
     public static class ProcessingTool
     {
+        #region Constants
+
+        /// <summary>
+        /// Shift-JIS encoding for detection and conversion
+        /// </summary>
+        private static readonly Encoding ShiftJIS = Encoding.GetEncoding(932);
+
+        #endregion
+
         #region Information Extraction
 
         /// <summary>
@@ -163,14 +172,15 @@ namespace MPF.Processors
             if (!File.Exists(filename))
                 return null;
 
+            // Read the entire file as bytes
+            byte[] bytes = File.ReadAllBytes(filename);
+
             // If we're reading as binary
             if (binary)
-            {
-                byte[] bytes = File.ReadAllBytes(filename);
                 return BitConverter.ToString(bytes).Replace("-", string.Empty);
-            }
 
-            return File.ReadAllText(filename);
+            // If we're reading as text
+            return NormalizeShiftJIS(bytes);
         }
 
         /// <summary>
@@ -292,6 +302,66 @@ namespace MPF.Processors
             return di.Units[0]?.Body?.DiscTypeIdentifier;
         }
 
+        /// <summary>
+        /// Normalize a byte array that may contain Shift-JIS characters
+        /// </summary>
+        /// <param name="contents">String as a byte array to normalize</param>
+        /// <returns>Normalized version of a string</returns>
+        public static string NormalizeShiftJIS(byte[]? contents)
+        {
+            // Invalid arrays are passed as-is
+            if (contents == null || contents.Length == 0)
+                return string.Empty;
+
+#if NET462_OR_GREATER || NETCOREAPP
+            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+#endif
+
+            // If the line contains Shift-JIS characters
+            if (BytesContainsShiftJIS(contents))
+                return ShiftJIS.GetString(contents);
+
+            return Encoding.UTF8.GetString(contents);
+        }
+
+        /// <summary>
+        /// Determine if a byte array contains Shift-JIS encoded characters
+        /// </summary>
+        /// <param name="line">Byte array to check for Shift-JIS encoding</param>
+        /// <returns>True if the byte array contains Shift-JIS characters, false otherwise</returns>
+        /// <see href="https://www.lemoda.net/c/detect-shift-jis/"/> 
+        internal static bool BytesContainsShiftJIS(byte[] bytes)
+        {
+            // Invalid arrays do not count
+            if (bytes == null || bytes.Length == 0)
+                return false;
+
+            // Loop through and check each pair of bytes
+            for (int i = 0; i < bytes.Length - 1; i++)
+            {
+                byte first = bytes[i];
+                byte second = bytes[i + 1];
+
+                if ((first >= 0x81 && first <= 0x84) ||
+                    (first >= 0x87 && first <= 0x9F))
+                {
+                    if (second >= 0x40 && second <= 0x9E)
+                        return true;
+                    else if (second >= 0x9F && second <= 0xFC)
+                        return true;
+                }
+                else if (first >= 0xE0 && first <= 0xEF)
+                {
+                    if (second >= 0x40 && second <= 0x9E)
+                        return true;
+                    else if (second >= 0x9F && second <= 0xFC)
+                        return true;
+                }
+            }
+
+            return false;
+        }
+
         #endregion
 
         #region Category Extraction

MPF.Processors/Redumper.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using System.Text.RegularExpressions;
 using SabreTools.Hashing;
 using SabreTools.RedumpLib;
@@ -116,7 +117,7 @@ namespace MPF.Processors
 
                     // Attempt to get the error count
                     long scsiErrors = GetSCSIErrorCount($"{basePath}.log");
-                    info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString());;
+                    info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString());
 
                     // Bluray-specific options
                     if (Type == MediaType.BluRay || Type == MediaType.NintendoWiiUOpticalDisc)
@@ -376,7 +377,7 @@ namespace MPF.Processors
                     string? ps2Protection = GetPlayStation2Protection($"{basePath}.log");
                     if (ps2Protection != null)
                         info.CommonDiscInfo!.Comments = $"<b>Protection</b>: {ps2Protection}" + Environment.NewLine;
-                    
+
                     break;
 
                 case RedumpSystem.SonyPlayStation3:
@@ -736,14 +737,16 @@ namespace MPF.Processors
                     return null;
 
                 // Now that we're at the relevant entries, read each line in and concatenate
-                string? cueString = string.Empty, line = sr.ReadLine()?.Trim();
+                var sb = new StringBuilder();
+                string? line = sr.ReadLine()?.Trim();
                 while (!string.IsNullOrEmpty(line))
                 {
-                    cueString += line + "\n";
+                    // TODO: Figure out how to use NormalizeShiftJIS here
+                    sb.AppendLine(line);
                     line = sr.ReadLine()?.Trim();
                 }
 
-                return cueString.TrimEnd('\n');
+                return sb.ToString().TrimEnd('\n');
             }
             catch
             {