Bump version to 0.8.2

Fixes issue where LiteralInlineParser calls PostMatch while processor.Inline is type other than LiteralInline

readme.md

@@ -46,6 +46,8 @@ You can **try Markdig online** and compare it to other implementations on [babel
   - **Emoji** support (inspired from [Markdown-it](https://markdown-it.github.io/))
   - **SmartyPants** (inspired from [Daring Fireball - SmartyPants](https://daringfireball.net/projects/smartypants/))
   - **Bootstrap** class (to output bootstrap class)
+  - **Diagrams** extension whenever a fenced code block contains a special keyword, it will be converted to a div block with the content as-is (currently, supports only for [`mermaid` diagrams](https://knsv.github.io/mermaid/))
+  - **YAML frontmatter** to parse without evaluating the frontmatter and to discard it from the HTML output (typically used for previewing without the frontmatter in MarkdownEditor)
 - Compatible with .NET 3.5, 4.0+ and .NET Core (`netstandard1.1+`)
 	
 ## Documentation

src/Markdig.Tests/Markdig.Tests.csproj

@@ -89,6 +89,7 @@
     <None Include="Specs\BootstrapSpecs.md" />
     <None Include="Specs\DiagramsSpecs.md" />
     <None Include="Specs\NoHtmlSpecs.md" />
+    <None Include="Specs\YamlSpecs.md" />
     <None Include="Specs\TaskListSpecs.md" />
     <None Include="Specs\SmartyPantsSpecs.md" />
     <None Include="Specs\MediaSpecs.md" />

src/Markdig.Tests/Specs/AbbreviationSpecs.md

@@ -57,3 +57,13 @@ We can abbreviate 1A, 1A1 and 1A2!
 .
 <p>We can abbreviate <abbr title="First">1A</abbr>, <abbr title="Second">1A1</abbr> and <abbr title="Third">1A2</abbr>!</p>
 ````````````````````````````````
+
+Abbreviations should match whole word only:
+
+```````````````````````````````` example
+*[1A]: First
+
+We should not abbreviate 1.1A or 11A!
+.
+<p>We should not abbreviate 1.1A or 11A!</p>
+````````````````````````````````

src/Markdig.Tests/Specs/Specs.tt

@@ -39,7 +39,7 @@ SOFTWARE.
 <#@ output extension=".cs" #><#
     var specFiles = new KeyValuePair<string, string>[] {
 		// new KeyValuePair<string, string>("https://raw.githubusercontent.com/jgm/CommonMark/master/spec.txt", string.Empty),
-		new KeyValuePair<string, string>("https://raw.githubusercontent.com/jgm/CommonMark/91e045ca370258903ed138450373043a496ec64b/spec.txt", string.Empty), // 0.26 specs
+		new KeyValuePair<string, string>("https://raw.githubusercontent.com/jgm/CommonMark/cfc84164475d3bec8be9482c21a705adc93a54f5/spec.txt", string.Empty), // 0.26 specs
 		new KeyValuePair<string, string>(Host.ResolvePath("PipeTableSpecs.md"), "pipetables|advanced"),
 		new KeyValuePair<string, string>(Host.ResolvePath("FootnotesSpecs.md"), "footnotes|advanced"),
 		new KeyValuePair<string, string>(Host.ResolvePath("GenericAttributesSpecs.md"), "attributes|advanced"),
@@ -60,6 +60,7 @@ SOFTWARE.
 		new KeyValuePair<string, string>(Host.ResolvePath("TaskListSpecs.md"), "tasklists|advanced"),
 		new KeyValuePair<string, string>(Host.ResolvePath("DiagramsSpecs.md"), "diagrams|advanced"),
 		new KeyValuePair<string, string>(Host.ResolvePath("NoHtmlSpecs.md"), "nohtml"),
+		new KeyValuePair<string, string>(Host.ResolvePath("YamlSpecs.md"), "yaml"),
 	};
     var emptyLines = false;
     var displayEmptyLines = false;

src/Markdig.Tests/Specs/YamlSpecs.md

@@ -0,0 +1,44 @@
+# Extensions
+
+Adds support for YAML frontmatter parsing:
+
+## YAML frontmatter discard
+ 
+If a frontmatter is present, it will not be rendered:
+
+```````````````````````````````` example
+---
+this: is a frontmatter
+---
+This is a text
+.
+<p>This is a text</p>
+````````````````````````````````
+ 
+But if a frontmatter doesn't happen on the first line, it will be parse as regular Markdown content
+
+```````````````````````````````` example
+This is a text1
+---
+this: is a frontmatter
+---
+This is a text2
+.
+<h2>This is a text1</h2>
+<h2>this: is a frontmatter</h2>
+<p>This is a text2</p>
+````````````````````````````````
+
+It expects an exact 3 dashes `---`:
+
+```````````````````````````````` example
+----
+this: is a frontmatter
+----
+This is a text
+.
+<hr />
+<h2>this: is a frontmatter</h2>
+<p>This is a text</p>
+````````````````````````````````
+

src/Markdig.Tests/TestPlayParser.cs

@@ -2,6 +2,9 @@
 // This file is licensed under the BSD-Clause 2 license. 
 // See the license.txt file in the project root for more information.
 using System;
+using System.Linq;
+using Markdig.Syntax;
+using Markdig.Syntax.Inlines;
 using NUnit.Framework;
 
 namespace Markdig.Tests
@@ -27,6 +30,17 @@ Later in a text we are using HTML and it becomes an abbr tag HTML
             Console.WriteLine(result);
         }
 
+        [Test]
+        public void TestEmptyLiteral()
+        {
+            var text = @"> *some text* 
+> some other text";
+            var doc = Markdown.Parse(text);
+
+            Assert.True(doc.Descendants().OfType<LiteralInline>().All(x => !x.Content.IsEmpty),
+                "There should not have any empty literals");
+        }
+
         [Test]
         public void TestSelfPipeline1()
         {
@@ -96,6 +110,12 @@ blabla
         }
 
 
+        [Test]
+        public void TestStandardUriEscape()
+        {
+            TestParser.TestSpec(@"![你好](你好.png)", "<p><img src=\"你好.png\" alt=\"你好\" /></p>", "nonascii-noescape");
+        }
+
 
         [Test]
         public void TestBugAdvancaed()

src/Markdig/Extensions/Abbreviations/AbbreviationParser.cs

@@ -101,20 +101,9 @@ namespace Markdig.Extensions.Abbreviations
                 for (int i = content.Start; i < content.End; i++)
                 {
                     string match;
-                    if (matcher.TryMatch(text, i, content.End - i + 1, out match))
+                    if (matcher.TryMatch(text, i, content.End - i + 1, out match) && IsValidAbbreviation(match, content, i))
                     {
-                        // The word matched must be embraced by punctuation or whitespace or \0.
-                        var c = content.PeekCharAbsolute(i - 1);
-                        if (!(c == '\0' || c.IsAsciiPunctuation() || c.IsWhitespace()))
-                        {
-                            continue;
-                        }
                         var indexAfterMatch = i + match.Length;
-                        c = content.PeekCharAbsolute(indexAfterMatch);
-                        if (!(c == '\0' || c.IsAsciiPunctuation() || c.IsWhitespace()))
-                        {
-                            continue;
-                        }
 
                         // We should have a match, but in case...
                         Abbreviation abbr;
@@ -196,5 +185,39 @@ namespace Markdig.Extensions.Abbreviations
                 }
             };
         }
+
+        private static bool IsValidAbbreviation(string match, StringSlice content, int matchIndex)
+        {
+            // The word matched must be embraced by punctuation or whitespace or \0.
+            var index = matchIndex - 1;
+            while (index > content.Start)
+            {
+                var c = content.PeekCharAbsolute(index);
+                if (!(c == '\0' || c.IsAsciiPunctuation() || c.IsWhitespace()))
+                {
+                    return false;
+                }
+                if (!c.IsAsciiPunctuation())
+                {
+                    break;
+                }
+                index--;
+            }
+            index = matchIndex + match.Length;
+            while (index < content.End)
+            {
+                var c = content.PeekCharAbsolute(index);
+                if (!(c == '\0' || c.IsAsciiPunctuation() || c.IsWhitespace()))
+                {
+                    return false;
+                }
+                if (!c.IsAsciiPunctuation())
+                {
+                    break;
+                }
+                index++;
+            }
+            return true;
+        }
     }
 }

src/Markdig/Extensions/DefinitionLists/HtmlDefinitionListRenderer.cs

@@ -49,7 +49,7 @@ namespace Markdig.Extensions.DefinitionLists
                     {
                         if (!hasOpendd)
                         {
-                            renderer.Write("<dd>");
+                            renderer.Write("<dd").WriteAttributes(definitionItem).Write(">");
                             countdd = 0;
                             hasOpendd = true;
                         }

src/Markdig/Extensions/NonAsciiNoEscape/NonAsciiNoEscapeExtension.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+
+using Markdig.Renderers;
+
+namespace Markdig.Extensions.NonAsciiNoEscape
+{
+    /// <summary>
+    /// Extension that will disable URI escape with % characters for non-US-ASCII characters in order to workaround a bug under IE/Edge with local file links containing non US-ASCII chars. DO NOT USE OTHERWISE.
+    /// </summary>
+    public class NonAsciiNoEscapeExtension : IMarkdownExtension
+    {
+        public void Setup(MarkdownPipelineBuilder pipeline)
+        {
+        }
+
+        public void Setup(IMarkdownRenderer renderer)
+        {
+            var htmlRenderer = renderer as HtmlRenderer;
+            if (htmlRenderer != null)
+            {
+                htmlRenderer.UseNonAsciiNoEscape = true;
+            }
+        }
+    }
+}

src/Markdig/Extensions/Yaml/YamlFrontMatterBlock.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+using Markdig.Parsers;
+using Markdig.Syntax;
+
+namespace Markdig.Extensions.Yaml
+{
+    /// <summary>
+    /// A YAML frontmatter block.
+    /// </summary>
+    /// <seealso cref="Markdig.Syntax.CodeBlock" />
+    public class YamlFrontMatterBlock : CodeBlock, IFencedBlock
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="YamlFrontMatterBlock"/> class.
+        /// </summary>
+        /// <param name="parser">The parser.</param>
+        public YamlFrontMatterBlock(BlockParser parser) : base(parser)
+        {
+        }
+
+        public string Info { get; set; }
+
+        public string Arguments { get; set; }
+
+        public int FencedCharCount { get; set; }
+
+        public char FencedChar { get; set; }
+    }
+}

src/Markdig/Extensions/Yaml/YamlFrontMatterExtension.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+
+using Markdig.Parsers;
+using Markdig.Renderers;
+using Markdig.Renderers.Html;
+
+namespace Markdig.Extensions.Yaml
+{
+    /// <summary>
+    /// Extension to discard a YAML frontmatter at the beginning of a Markdown document.
+    /// </summary>
+    public class YamlFrontMatterExtension : IMarkdownExtension
+    {
+        public void Setup(MarkdownPipelineBuilder pipeline)
+        {
+            if (!pipeline.BlockParsers.Contains<YamlFrontMatterParser>())
+            {
+                // Insert the YAML parser before the thematic break parser, as it is also triggered on a --- dash
+                pipeline.BlockParsers.InsertBefore<ThematicBreakParser>(new YamlFrontMatterParser());
+            }
+        }
+
+        public void Setup(IMarkdownRenderer renderer)
+        {
+            if (!renderer.ObjectRenderers.Contains<YamlFrontMatterRenderer>())
+            {
+                renderer.ObjectRenderers.InsertBefore<CodeBlockRenderer>(new YamlFrontMatterRenderer());
+            }
+        }
+    }
+}

src/Markdig/Extensions/Yaml/YamlFrontMatterParser.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+using Markdig.Parsers;
+
+namespace Markdig.Extensions.Yaml
+{
+    /// <summary>
+    /// Block parser for a YAML frontmatter. 
+    /// </summary>
+    /// <seealso cref="Markdig.Parsers.FencedBlockParserBase{YamlFrontMatterBlock}" />
+    public class YamlFrontMatterParser : FencedBlockParserBase<YamlFrontMatterBlock>
+    {
+        // We reuse a FencedCodeBlock parser to grab a frontmatter, only active if it happens on the first line of the document.
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FencedCodeBlockParser"/> class.
+        /// </summary>
+        public YamlFrontMatterParser()
+        {
+            OpeningCharacters = new[] { '-' };
+            InfoPrefix = null;
+            // We expect only 3 --- at the beginning of the file no more, no less
+            MinimumMatchCount = 3;
+            MaximumMatchCount = 3;
+        }
+
+        protected override YamlFrontMatterBlock CreateFencedBlock(BlockProcessor processor)
+        {
+            return new YamlFrontMatterBlock(this);
+        }
+
+        public override BlockState TryOpen(BlockProcessor processor)
+        {
+            // Only accept a frontmatter at the beginning of the file
+            if (processor.LineIndex != 0)
+            {
+                return BlockState.None;
+            }
+
+            return base.TryOpen(processor);
+        }
+    }
+}

src/Markdig/Extensions/Yaml/YamlFrontMatterRenderer.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+using Markdig.Renderers;
+using Markdig.Renderers.Html;
+
+namespace Markdig.Extensions.Yaml
+{
+    /// <summary>
+    /// Empty renderer for a <see cref="YamlFrontMatterBlock"/>
+    /// </summary>
+    /// <seealso cref="Markdig.Renderers.Html.HtmlObjectRenderer{YamlFrontMatterBlock}" />
+    public class YamlFrontMatterRenderer : HtmlObjectRenderer<YamlFrontMatterBlock>
+    {
+        protected override void Write(HtmlRenderer renderer, YamlFrontMatterBlock obj)
+        {
+        }
+    }
+}

src/Markdig/MarkdownExtensions.cs

@@ -24,8 +24,10 @@ using Markdig.Extensions.NoRefLinks;
 using Markdig.Extensions.PragmaLines;
 using Markdig.Extensions.SelfPipeline;
 using Markdig.Extensions.SmartyPants;
+using Markdig.Extensions.NonAsciiNoEscape;
 using Markdig.Extensions.Tables;
 using Markdig.Extensions.TaskLists;
+using Markdig.Extensions.Yaml;
 using Markdig.Parsers;
 using Markdig.Parsers.Inlines;
 
@@ -63,6 +65,28 @@ namespace Markdig
                 .UseGenericAttributes(); // Must be last as it is one parser that is modifying other parsers
         }
 
+        /// <summary>
+        /// Uses this extension to disable URI escape with % characters for non-US-ASCII characters in order to workaround a bug under IE/Edge with local file links containing non US-ASCII chars. DO NOT USE OTHERWISE.
+        /// </summary>
+        /// <param name="pipeline">The pipeline.</param>
+        /// <returns>The modified pipeline</returns>
+        public static MarkdownPipelineBuilder UseNonAsciiNoEscape(this MarkdownPipelineBuilder pipeline)
+        {
+            pipeline.Extensions.AddIfNotAlready<NonAsciiNoEscapeExtension>();
+            return pipeline;
+        }
+
+        /// <summary>
+        /// Uses YAML frontmatter extension that will parse a YAML frontmatter into the MarkdownDocument. Note that they are not rendered by any default HTML renderer.
+        /// </summary>
+        /// <param name="pipeline">The pipeline.</param>
+        /// <returns>The modified pipeline</returns>
+        public static MarkdownPipelineBuilder UseYamlFrontMatter(this MarkdownPipelineBuilder pipeline)
+        {
+            pipeline.Extensions.AddIfNotAlready<YamlFrontMatterExtension>();
+            return pipeline;
+        }
+
         /// <summary>
         /// Uses the self pipeline extension that will detect the pipeline to use from the markdown input that contains a special tag. See <see cref="SelfPipelineExtension"/>
         /// </summary>
@@ -412,6 +436,8 @@ namespace Markdig
                 return pipeline;
             }
 
+            // TODO: the extension string should come from the extension itself instead of this hardcoded switch case.
+
             foreach (var extension in extensions.Split(new[] { '+' }, StringSplitOptions.RemoveEmptyEntries))
             {
                 switch (extension.ToLowerInvariant())
@@ -490,6 +516,12 @@ namespace Markdig
                     case "nohtml":
                         pipeline.DisableHtml();
                         break;
+                    case "yaml":
+                        pipeline.UseYamlFrontMatter();
+                        break;
+                    case "nonascii-noescape":
+                        pipeline.UseNonAsciiNoEscape();
+                        break;
                     default:
                         throw new ArgumentException($"Invalid extension `{extension}` from `{extensions}`", nameof(extensions));
                 }

src/Markdig/Parsers/Inlines/LinkInlineParser.cs

@@ -211,6 +211,7 @@ namespace Markdig.Parsers.Inlines
                 // compact reference link/image, 
                 // or shortcut reference link/image
                 var parentDelimiter = openParent.Parent;
+                var savedText = text;
                 switch (text.CurrentChar)
                 {
                     case '(':
@@ -253,9 +254,10 @@ namespace Markdig.Parsers.Inlines
 
                             return true;
                         }
-                        break;
-                    default:
 
+                        text = savedText;
+                        goto default;
+                    default:
                         var labelSpan = SourceSpan.Empty;
                         string label = null;
                         bool isLabelSpanLocal = true;

src/Markdig/Parsers/Inlines/LiteralInlineParser.cs

@@ -75,21 +75,26 @@ namespace Markdig.Parsers.Inlines
             }
             else
             {
-                processor.Inline = new LiteralInline()
+                // Create a new LiteralInline only if it is not empty
+                var newSlice = length > 0 ? new StringSlice(slice.Text, slice.Start, endPosition) : StringSlice.Empty;
+                if (!newSlice.IsEmpty)
                 {
-                    Content = length > 0 ? new StringSlice(slice.Text, slice.Start, endPosition) : StringSlice.Empty,
-                    Span = new SourceSpan(startPosition, processor.GetSourcePosition(endPosition)),
-                    Line = line,
-                    Column = column,
-                };
+                    processor.Inline = new LiteralInline()
+                    {
+                        Content = length > 0 ? newSlice : StringSlice.Empty,
+                        Span = new SourceSpan(startPosition, processor.GetSourcePosition(endPosition)),
+                        Line = line,
+                        Column = column,
+                    };
+                }
             }
 
             slice.Start = nextStart;
 
             // Call only PostMatch if necessary
-            if (PostMatch != null)
+            if (processor.Inline is LiteralInline)
             {
-                PostMatch(processor, ref slice);
+                PostMatch?.Invoke(processor, ref slice);
             }
 
             return true;

src/Markdig/Properties/AssemblyInfo.cs

@@ -25,6 +25,6 @@ namespace Markdig
 {
     public static partial class Markdown
     {
-        public const string Version = "0.7.5";
+        public const string Version = "0.8.2";
     }
 }

src/Markdig/Renderers/HtmlRenderer.cs

@@ -64,6 +64,8 @@ namespace Markdig.Renderers
         /// </summary>
         public bool ImplicitParagraph { get; set; }
 
+        public bool UseNonAsciiNoEscape { get; set; }
+
         /// <summary>
         /// Writes the content escaped for HTML.
         /// </summary>
@@ -203,22 +205,29 @@ namespace Markdig.Renderers
                     Write(content, previousPosition, i - previousPosition);
                     previousPosition = i + 1;
 
-                    byte[] bytes;
-                    if (c >= '\ud800' && c <= '\udfff' && previousPosition < length)
+                    // Special case for Edge/IE workaround for MarkdownEditor, don't escape non-ASCII chars to make image links working
+                    if (UseNonAsciiNoEscape)
                     {
-                        bytes = Encoding.UTF8.GetBytes(new[] { c, content[previousPosition] });
-                        // Skip next char as it is decoded above
-                        i++;
-                        previousPosition = i + 1;
+                        Write(c);
                     }
                     else
                     {
-                        bytes = Encoding.UTF8.GetBytes(new[] { c });
-                    }
-
-                    for (var j = 0; j < bytes.Length; j++)
-                    {
-                        Write($"%{bytes[j]:X2}");
+                        byte[] bytes;
+                        if (c >= '\ud800' && c <= '\udfff' && previousPosition < length)
+                        {
+                            bytes = Encoding.UTF8.GetBytes(new[] { c, content[previousPosition] });
+                            // Skip next char as it is decoded above
+                            i++;
+                            previousPosition = i + 1;
+                        }
+                        else
+                        {
+                            bytes = Encoding.UTF8.GetBytes(new[] { c });
+                        }
+                        for (var j = 0; j < bytes.Length; j++)
+                        {
+                            Write($"%{bytes[j]:X2}");
+                        }
                     }
                 }
             }

src/Markdig/project.json

@@ -1,6 +1,6 @@
 {
   "title": "Markdig",
-  "version": "0.7.5",
+  "version": "0.8.2",
   "authors": [ "Alexandre Mutel" ],
   "description": "A fast, powerfull, CommonMark compliant, extensible Markdown processor for .NET with 20+ builtin extensions (pipetables, footnotes, definition lists... etc.)",
   "copyright": "Alexandre Mutel",
@@ -11,7 +11,7 @@
     "projectUrl": "https://github.com/lunet-io/markdig",
     "iconUrl": "https://raw.githubusercontent.com/lunet-io/markdig/master/img/markdig.png",
     "requireLicenseAcceptance": false,
-    "releaseNotes": "> 0.7.5\n- several bug fixes (pipe tables, disable HTML, special attributes, inline maths, abbreviations...)\n- add support for rowspan in grid tables\n> 0.7.4\n- Fix bug with strong emphasis starting at the beginning of a line\n> 0.7.3\n- Fix threading issue with pipeline\n> 0.7.2\n- Fix rendering of table colspan with non english locale\n- Fix grid table colspan parsing\n- Add nofollow extension for links\n> 0.7.1\n- Fix issue in smarty pants which could lead to an InvalidCastException\n- Update parsers to latest CommonMark specs\n>0.7.0\n- Update to latest NETStandard.Library 1.6.0\n- Fix issue with digits in auto-identifier extension\n- Fix incorrect start of span calculated for code indented blocks",
+    "releaseNotes": "> 0.8.2\n- fix potential cast exception with Abreviation extension and empty literals\n> 0.8.1\n- new extension to disable URI escaping for non-US-ASCII characters to workaround a bug in Edge/IE\n- Fix an issue with abbreviations with left/right multiple non-punctuation/space characters\n> 0.8.0\n- Update to latest CommonMark specs\n- Fix empty literal\n- Add YAML frontmatter extension\n",
     "tags": [ "Markdown CommonMark md html md2html" ]
   },
   "configurations": {