using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Net; using System.Text; namespace UnicodeNormDApp { class Program { static void Main(string[] args) { var httpClient = new WebClient(); var data = httpClient.DownloadString("http://www.unicode.org/Public/UCD/latest/ucd/NormalizationTest.txt"); var stringReader = new StringReader(data); var sep = new char[] {';'}; var spaceSpec = new char[] {' '}; string line; int count = 0; int min = int.MaxValue; int max = int.MinValue; var values = new Dictionary(); var builder = new StringBuilder(); while ((line = stringReader.ReadLine()) != null) { if (line.StartsWith("#") || line.StartsWith("@")) { continue; } var commentIndex = line.IndexOf('#'); var dataLine = commentIndex > 0 ? line.Substring(0, commentIndex) : line; var columns = dataLine.Split(sep, StringSplitOptions.RemoveEmptyEntries); if (columns.Length < 4) { continue; } // Skip multi code point if (columns[0].IndexOf(' ') > 0) { continue; } var source = Convert.ToInt32(columns[0], 16); if (source < min) { min = source; } if (source > max) { max = source; } var column4Space = columns[4].Split(spaceSpec, StringSplitOptions.RemoveEmptyEntries); builder.Clear(); for (int i = 0; i < column4Space.Length; i++) { var nfdFirst = Convert.ToInt32(column4Space[i], 16); // We support only single char codepoints string unicodeString = char.ConvertFromUtf32(nfdFirst); // We restrict to ascii only if (unicodeString.Length == 1 && nfdFirst > 32 && nfdFirst < 127) { builder.Append(unicodeString[0]); } } var str = builder.ToString(); var sourceString = char.ConvertFromUtf32(source); // We don't keep spaces if (sourceString.Length == 1 && str.Length > 0 && !values.ContainsKey(sourceString[0])) { //Trace.WriteLine(columns[0] + "/" + source + ": " + char.ConvertFromUtf32(source) + " => " + (char)nfdFirst); count++; values.Add(sourceString[0], str); } } //var newValues = new Dictionary(values.Count) //{ // {15, 'a'} //} Trace.WriteLine($"CodeToAscii = new Dictionary({values.Count})"); Trace.WriteLine("{"); foreach (var pair in values) { var escape = pair.Value.Replace("\\", @"\\").Replace("\"", "\\\""); Trace.WriteLine($" {{'{pair.Key}',\"{escape}\"}},"); } Trace.WriteLine("};"); //Trace.WriteLine("count: " + count); //Trace.WriteLine("max: " + max); //Trace.WriteLine("min: " + min); } } }