[Style, DatFile] Add better encoding finding

SabreTools.Helper/Objects/Dat/DatFile.cs

@@ -707,7 +707,8 @@ namespace SabreTools.Helper
 			bool clean)
 		{
 			// Open a file reader
-			StreamReader sr = new StreamReader(File.OpenRead(filename));
+			Encoding enc = Style.GetEncoding(filename);
+			StreamReader sr = new StreamReader(File.OpenRead(filename), enc);
 
 			bool block = false, superdat = false;
 			string blockname = "", tempgamename = "", gamedesc = "", cloneof = "",
@@ -1202,7 +1203,8 @@ namespace SabreTools.Helper
 			bool clean)
 		{
 			// Open a file reader
-			StreamReader sr = new StreamReader(File.OpenRead(filename));
+			Encoding enc = Style.GetEncoding(filename);
+			StreamReader sr = new StreamReader(File.OpenRead(filename), enc);
 
 			string blocktype = "";
 			while (!sr.EndOfStream)
@@ -2249,22 +2251,8 @@ namespace SabreTools.Helper
 			List<string> keys = dict.Keys.ToList();
 			foreach (string key in keys)
 			{
-				// If the dictionary somehow doesn't have the key in question, continue
-				if (!dict.ContainsKey(key))
-				{
-					logger.Warning("Input Dictionary does not contain key: " + key);
-					continue;
-				}
-
 				List<DatItem> roms = dict[key];
 
-				// If we somehow have a null list, just skip it
-				if (roms == null)
-				{
-					logger.Warning("Blank list found for key: " + key);
-					continue;
-				}
-
 				// If we're merging the roms, do so
 				if (mergeroms)
 				{
@@ -2282,6 +2270,7 @@ namespace SabreTools.Helper
 						+ (String.IsNullOrEmpty(rom.MachineName)
 								? "Default"
 								: rom.MachineName.ToLowerInvariant());
+					newkey = HttpUtility.HtmlEncode(newkey);
 					if (sortable.ContainsKey(newkey))
 					{
 						sortable[newkey].Add(rom);
@@ -2306,13 +2295,6 @@ namespace SabreTools.Helper
 					continue;
 				}
 
-				// If we somehow have a null list, just skip it
-				if (sortable[key] == null)
-				{
-					logger.Warning("Blank list found for key: " + key);
-					continue;
-				}
-
 				List<DatItem> sortedlist = sortable[key];
 				DatItem.Sort(ref sortedlist, norename);
 				sortable[key] = sortedlist;
@@ -3012,13 +2994,6 @@ namespace SabreTools.Helper
 							continue;
 						}
 
-						// If we somehow have a null list, just skip it
-						if (sortable[key] == null)
-						{
-							logger.Warning("Blank list found for key: " + key);
-							continue;
-						}
-
 						List<DatItem> roms = sortable[key];
 
 						for (int index = 0; index < roms.Count; index++)

SabreTools.Helper/Tools/Style.cs

@@ -576,6 +576,31 @@ namespace SabreTools.Helper
 			return false;
 		}
 
+		/// <summary>
+		/// Determines a text file's encoding by analyzing its byte order mark (BOM).
+		/// Defaults to ASCII when detection of the text file's endianness fails.
+		/// http://stackoverflow.com/questions/3825390/effective-way-to-find-any-files-encoding
+		/// </summary>
+		/// <param name="filename">The text file to analyze.</param>
+		/// <returns>The detected encoding.</returns>
+		public static Encoding GetEncoding(string filename)
+		{
+			// Read the BOM
+			var bom = new byte[4];
+			using (var file = new FileStream(filename, FileMode.Open, FileAccess.Read))
+			{
+				file.Read(bom, 0, 4);
+			}
+
+			// Analyze the BOM
+			if (bom[0] == 0x2b && bom[1] == 0x2f && bom[2] == 0x76) return Encoding.UTF7;
+			if (bom[0] == 0xef && bom[1] == 0xbb && bom[2] == 0xbf) return Encoding.UTF8;
+			if (bom[0] == 0xff && bom[1] == 0xfe) return Encoding.Unicode; //UTF-16LE
+			if (bom[0] == 0xfe && bom[1] == 0xff) return Encoding.BigEndianUnicode; //UTF-16BE
+			if (bom[0] == 0 && bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff) return Encoding.UTF32;
+			return Encoding.Default;
+		}
+
 		#endregion
 	}
 }