diff --git a/CHANGELIST.md b/CHANGELIST.md
index e43bfb75..b55f7def 100644
--- a/CHANGELIST.md
+++ b/CHANGELIST.md
@@ -35,6 +35,7 @@
- Update RedumpLib to 1.6.5
- Fix package reference layout
- Update Nuget packages
+- Normalize Shift-JIS characters, when possible
### 3.3.0 (2025-01-03)
diff --git a/MPF.Processors.Test/ProcessingToolTests.cs b/MPF.Processors.Test/ProcessingToolTests.cs
index 108c6134..d6efb2d4 100644
--- a/MPF.Processors.Test/ProcessingToolTests.cs
+++ b/MPF.Processors.Test/ProcessingToolTests.cs
@@ -426,6 +426,46 @@ namespace MPF.Processors.Test
#endregion
+ #region NormalizeShiftJIS
+
+ [Fact]
+ public void NormalizeShiftJIS_Null_Empty()
+ {
+ byte[]? contents = null;
+ string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+ Assert.NotNull(actual);
+ Assert.Empty(actual);
+ }
+
+ [Fact]
+ public void NormalizeShiftJIS_Empty_Empty()
+ {
+ byte[]? contents = [];
+ string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+ Assert.NotNull(actual);
+ Assert.Empty(actual);
+ }
+
+ [Fact]
+ public void NormalizeShiftJIS_NoShiftJIS_Valid()
+ {
+ string? expected = "ABCDE";
+ byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45];
+ string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+ Assert.Equal(expected, actual);
+ }
+
+ [Fact]
+ public void NormalizeShiftJIS_ShiftJIS_Valid()
+ {
+ string? expected = "ABCDE ひらがな";
+ byte[]? contents = [0x41, 0x42, 0x43, 0x44, 0x45, 0x20, 0x82, 0xD0, 0x82, 0xE7, 0x82, 0xAA, 0x82, 0xC8];
+ string? actual = ProcessingTool.NormalizeShiftJIS(contents);
+ Assert.Equal(expected, actual);
+ }
+
+ #endregion
+
#region GetUMDCategory
[Theory]
diff --git a/MPF.Processors/ProcessingTool.cs b/MPF.Processors/ProcessingTool.cs
index 6ac38a26..b768c54e 100644
--- a/MPF.Processors/ProcessingTool.cs
+++ b/MPF.Processors/ProcessingTool.cs
@@ -22,6 +22,15 @@ namespace MPF.Processors
///
public static class ProcessingTool
{
+ #region Constants
+
+ ///
+ /// Shift-JIS encoding for detection and conversion
+ ///
+ private static readonly Encoding ShiftJIS = Encoding.GetEncoding(932);
+
+ #endregion
+
#region Information Extraction
///
@@ -163,14 +172,15 @@ namespace MPF.Processors
if (!File.Exists(filename))
return null;
+ // Read the entire file as bytes
+ byte[] bytes = File.ReadAllBytes(filename);
+
// If we're reading as binary
if (binary)
- {
- byte[] bytes = File.ReadAllBytes(filename);
return BitConverter.ToString(bytes).Replace("-", string.Empty);
- }
- return File.ReadAllText(filename);
+ // If we're reading as text
+ return NormalizeShiftJIS(bytes);
}
///
@@ -292,6 +302,66 @@ namespace MPF.Processors
return di.Units[0]?.Body?.DiscTypeIdentifier;
}
+ ///
+ /// Normalize a byte array that may contain Shift-JIS characters
+ ///
+ /// String as a byte array to normalize
+ /// Normalized version of a string
+ public static string NormalizeShiftJIS(byte[]? contents)
+ {
+ // Invalid arrays are passed as-is
+ if (contents == null || contents.Length == 0)
+ return string.Empty;
+
+#if NET462_OR_GREATER || NETCOREAPP
+ Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+#endif
+
+ // If the line contains Shift-JIS characters
+ if (BytesContainsShiftJIS(contents))
+ return ShiftJIS.GetString(contents);
+
+ return Encoding.UTF8.GetString(contents);
+ }
+
+ ///
+ /// Determine if a byte array contains Shift-JIS encoded characters
+ ///
+ /// Byte array to check for Shift-JIS encoding
+ /// True if the byte array contains Shift-JIS characters, false otherwise
+ ///
+ internal static bool BytesContainsShiftJIS(byte[] bytes)
+ {
+ // Invalid arrays do not count
+ if (bytes == null || bytes.Length == 0)
+ return false;
+
+ // Loop through and check each pair of bytes
+ for (int i = 0; i < bytes.Length - 1; i++)
+ {
+ byte first = bytes[i];
+ byte second = bytes[i + 1];
+
+ if ((first >= 0x81 && first <= 0x84) ||
+ (first >= 0x87 && first <= 0x9F))
+ {
+ if (second >= 0x40 && second <= 0x9E)
+ return true;
+ else if (second >= 0x9F && second <= 0xFC)
+ return true;
+ }
+ else if (first >= 0xE0 && first <= 0xEF)
+ {
+ if (second >= 0x40 && second <= 0x9E)
+ return true;
+ else if (second >= 0x9F && second <= 0xFC)
+ return true;
+ }
+ }
+
+ return false;
+ }
+
#endregion
#region Category Extraction
diff --git a/MPF.Processors/Redumper.cs b/MPF.Processors/Redumper.cs
index 545a7268..9d18b6d1 100644
--- a/MPF.Processors/Redumper.cs
+++ b/MPF.Processors/Redumper.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
+using System.Text;
using System.Text.RegularExpressions;
using SabreTools.Hashing;
using SabreTools.RedumpLib;
@@ -116,7 +117,7 @@ namespace MPF.Processors
// Attempt to get the error count
long scsiErrors = GetSCSIErrorCount($"{basePath}.log");
- info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString());;
+ info.CommonDiscInfo!.ErrorsCount = (scsiErrors == -1 ? "Error retrieving error count" : scsiErrors.ToString());
// Bluray-specific options
if (Type == MediaType.BluRay || Type == MediaType.NintendoWiiUOpticalDisc)
@@ -376,7 +377,7 @@ namespace MPF.Processors
string? ps2Protection = GetPlayStation2Protection($"{basePath}.log");
if (ps2Protection != null)
info.CommonDiscInfo!.Comments = $"Protection: {ps2Protection}" + Environment.NewLine;
-
+
break;
case RedumpSystem.SonyPlayStation3:
@@ -736,14 +737,16 @@ namespace MPF.Processors
return null;
// Now that we're at the relevant entries, read each line in and concatenate
- string? cueString = string.Empty, line = sr.ReadLine()?.Trim();
+ var sb = new StringBuilder();
+ string? line = sr.ReadLine()?.Trim();
while (!string.IsNullOrEmpty(line))
{
- cueString += line + "\n";
+ // TODO: Figure out how to use NormalizeShiftJIS here
+ sb.AppendLine(line);
line = sr.ReadLine()?.Trim();
}
- return cueString.TrimEnd('\n');
+ return sb.ToString().TrimEnd('\n');
}
catch
{