Bump version to 0.5.9

readme.md

@@ -10,12 +10,13 @@ You can **try Markdig online** and compare it to other implementations on [babel
 
 ## Features
 
-- **Very fast parser** (no-regexp), very lightweight in terms of GC pressure. See benchmarks
-- **Abstract Syntax Tree**
+- **Very fast parser and html renderer** (no-regexp), very lightweight in terms of GC pressure. See benchmarks
+- **Abstract Syntax Tree** with precise source code location for syntax tree, useful when building a Markdown editor.
+  - Checkout [MarkdownEditor for Visual Studio](https://visualstudiogallery.msdn.microsoft.com/eaab33c3-437b-4918-8354-872dfe5d1bfe) powered by Markdig!
 - Converter to **HTML**
 - Passing more than **600+ tests** from the latest [CommonMark specs](http://spec.commonmark.org/)
 - Includes all the core elements of CommonMark:
-  - including GFM fenced code blocks. 
+  - including **GFM fenced code blocks**.  
 - **Extensible** architecture
   - Even the core Markdown/CommonMark parsing is pluggable, so it allows to disable builtin Markdown/Commonmark parsing (e.g [Disable HTML parsing](https://github.com/lunet-io/markdig/blob/7964bd0160d4c18e4155127a4c863d61ebd8944a/src/Markdig/MarkdownExtensions.cs#L306)) or change behaviour (e.g change matching `#` of a headers with `@`)   
 - Built-in with **20+ extensions**, including:

src/Markdig.Tests/Markdig.Tests.csproj

@@ -62,6 +62,7 @@
     <Compile Include="TestHtmlHelper.cs" />
     <Compile Include="TestLineReader.cs" />
     <Compile Include="TestLinkHelper.cs" />
+    <Compile Include="TestPragmaLines.cs" />
     <Compile Include="TestSourcePosition.cs" />
     <Compile Include="TestStringSliceList.cs" />
     <Compile Include="TestPlayParser.cs" />

src/Markdig.Tests/TestPlayParser.cs

@@ -27,6 +27,41 @@ Later in a text we are using HTML and it becomes an abbr tag HTML
             Console.WriteLine(result);
         }
 
+        [Test]
+        public void TestListBug()
+        {
+            // TODO: Add this test back to the CommonMark specs
+            var text = @"- item1
+  - item2
+    - item3
+      - item4";
+            TestParser.TestSpec(text, @"<ul>
+<li>item1
+<ul>
+<li>item2
+<ul>
+<li>item3
+<ul>
+<li>item4</li>
+</ul></li>
+</ul></li>
+</ul></li>
+</ul>
+");
+        }
+
+
+
+
+        [Test]
+        public void TestBugAdvancaed()
+        {
+            TestParser.TestSpec(@"`https://{domain}/callbacks`
+#### HEADING
+Paragraph
+", "<p><code>https://{domain}/callbacks</code></p>\n<h4 id=\"heading\">HEADING</h4>\n<p>Paragraph</p>", "advanced");
+        }
+
         [Test]
         public void TestSamePipelineAllExtensions()
         {

src/Markdig.Tests/TestPragmaLines.cs

@@ -0,0 +1,81 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+
+using Markdig.Renderers;
+using Markdig.Syntax;
+using NUnit.Framework;
+
+namespace Markdig.Tests
+{
+    [TestFixture]
+    public class TestPragmaLines
+    {
+        [Test]
+        public void TestFindClosest()
+        {
+            var doc = Markdown.Parse(
+"test1\n" +                      // 0
+"\n" +                           // 1
+"test2\n" +                      // 2
+"\n" +                           // 3
+"test3\n" +                      // 4
+"\n" +                           // 5
+"test4\n" +                      // 6
+"\n" +                           // 7
+"# Heading\n" +                  // 8
+"\n" +                           // 9
+"Long para\n" +                  // 10
+"on multiple\n" +                // 11
+"lines\n" +                      // 12
+"to check that\n" +              // 13
+"lines are\n" +                  // 14
+"correctly \n" +                 // 15
+"found\n" +                      // 16
+"\n" +                           // 17
+"- item1\n" +                    // 18
+"- item2\n" +                    // 19
+"- item3\n" +                    // 20
+"\n" +                           // 21
+"This is a last paragraph\n"     // 22
+                , new MarkdownPipelineBuilder().UsePragmaLines().Build());
+
+            foreach (var exact in new int[] {0, 2, 4, 6, 8, 10, 18, 19, 20, 22})
+            {
+                Assert.AreEqual(exact, doc.FindClosestLine(exact));
+            }
+
+            Assert.AreEqual(22, doc.FindClosestLine(23));
+
+            Assert.AreEqual(10, doc.FindClosestLine(11));
+            Assert.AreEqual(10, doc.FindClosestLine(12));
+            Assert.AreEqual(10, doc.FindClosestLine(13));
+            Assert.AreEqual(18, doc.FindClosestLine(14)); // > 50% of the paragraph, we switch to next
+            Assert.AreEqual(18, doc.FindClosestLine(15));
+            Assert.AreEqual(18, doc.FindClosestLine(16));
+        }
+
+        [Test]
+        public void TestFindClosest1()
+        {
+            var text = 
+"- item1\n" +                    // 0
+"  - item11\n" +                 // 1
+"  - item12\n" +                 // 2
+"    - item121\n" +              // 3
+"  - item13\n" +                 // 4
+"    - item131\n" +              // 5
+"      - item1311\n";            // 6
+
+            var pipeline = new MarkdownPipelineBuilder().UsePragmaLines().Build();
+            var doc = Markdown.Parse(text, pipeline);
+
+            for (int exact = 0; exact < 7; exact++)
+            {
+                Assert.AreEqual(exact, doc.FindClosestLine(exact));
+            }
+
+            Assert.AreEqual(6, doc.FindClosestLine(50));
+        }
+    }
+}

src/Markdig/Extensions/DefinitionLists/DefinitionListParser.cs

@@ -98,7 +98,7 @@ namespace Markdig.Extensions.DefinitionLists
             processor.Open(definitionItem);
 
             // Update the end position
-            currentDefinitionList.Span.End = processor.Line.End;
+            currentDefinitionList.UpdateSpanEnd(processor.Line.End);
 
             return BlockState.Continue;
         }

src/Markdig/Extensions/Figures/FigureBlockParser.cs

@@ -114,7 +114,7 @@ namespace Markdig.Extensions.Figures
                     figure.Add(caption);
                 }
 
-                figure.Span.End = line.End;
+                figure.UpdateSpanEnd(line.End);
 
                 // Don't keep the last line
                 return BlockState.BreakDiscard;
@@ -123,7 +123,7 @@ namespace Markdig.Extensions.Figures
             // Reset the indentation to the column before the indent
             processor.GoToColumn(processor.ColumnBeforeIndent);
 
-            figure.Span.End = line.End;
+            figure.UpdateSpanEnd(line.End);
 
             return BlockState.Continue;
         }

src/Markdig/Extensions/Footers/FooterBlockParser.cs

@@ -80,7 +80,7 @@ namespace Markdig.Extensions.Footers
                 {
                     processor.NextChar(); // Skip following space
                 }
-                block.Span.End = processor.Line.End;
+                block.UpdateSpanEnd(processor.Line.End);
             }
             return result;
         }

src/Markdig/Extensions/GenericAttributes/GenericAttributesExtension.cs

@@ -46,7 +46,7 @@ namespace Markdig.Extensions.GenericAttributes
             // Try to find if there is any attributes { in the info string on the first line of a FencedCodeBlock
             if (line.Start < line.End)
             {
-                var indexOfAttributes = line.Text.LastIndexOf('{', line.End);
+                int indexOfAttributes = line.IndexOf('{');
                 if (indexOfAttributes >= 0)
                 {
                     // Work on a copy

src/Markdig/Extensions/PragmaLines/PragmaLineExtension.cs

@@ -2,8 +2,12 @@
 // This file is licensed under the BSD-Clause 2 license. 
 // See the license.txt file in the project root for more information.
 
+using System;
+using Markdig.Helpers;
 using Markdig.Renderers;
+using Markdig.Renderers.Html;
 using Markdig.Syntax;
+using Markdig.Syntax.Inlines;
 
 namespace Markdig.Extensions.PragmaLines
 {
@@ -15,26 +19,61 @@ namespace Markdig.Extensions.PragmaLines
     {
         public void Setup(MarkdownPipelineBuilder pipeline)
         {
+            pipeline.DocumentProcessed -= PipelineOnDocumentProcessed;
+            pipeline.DocumentProcessed += PipelineOnDocumentProcessed;
         }
 
         public void Setup(IMarkdownRenderer renderer)
         {
-            var htmlRenderer = renderer as HtmlRenderer;
-            if (htmlRenderer != null)
+        }
+
+        private static void PipelineOnDocumentProcessed(MarkdownDocument document)
+        {
+            int index = 0;
+            AddPragmas(document, ref index);
+        }
+
+        private static void AddPragmas(Block block, ref int index)
+        {
+            var attribute = block.GetAttributes();
+            var pragmaId = GetPragmaId(block);
+            if ( attribute.Id == null)
             {
-                htmlRenderer.ObjectWriteBefore -= HtmlRendererOnObjectWriteBefore;
-                htmlRenderer.ObjectWriteBefore += HtmlRendererOnObjectWriteBefore;
+                attribute.Id = pragmaId;
+            }
+            else if (block.Parent != null)
+            {
+                var heading = block as HeadingBlock;
+
+                // If we have a heading, we will try to add the tag inside it
+                // otherwise we will add it just before
+                var tag = $"<a id=\"{pragmaId}\"></a>";
+                if (heading?.Inline?.FirstChild != null)
+                {
+                    heading.Inline.FirstChild.InsertBefore(new HtmlInline() { Tag = tag });
+
+                }
+                else
+                {
+                    block.Parent.Insert(index, new HtmlBlock(null) { Lines = new StringLineGroup(tag) });
+                    index++;
+                }
+            }
+
+            var container = block as ContainerBlock;
+            if (container != null)
+            {
+                for (int i = 0; i < container.Count; i++)
+                {
+                    var subBlock = container[i];
+                    AddPragmas(subBlock, ref i);
+                }
             }
         }
 
-        private static void HtmlRendererOnObjectWriteBefore(IMarkdownRenderer renderer, MarkdownObject markdownObject)
+        private static string GetPragmaId(Block block)
         {
-            if (markdownObject is Block)
-            {
-                var htmlRenderer = (HtmlRenderer) renderer;
-                htmlRenderer.EnsureLine();
-                htmlRenderer.WriteLine($"<span id=\"pragma-line-{markdownObject.Line}\"></span>");
-            }
+            return $"pragma-line-{block.Line}";
         }
     }
 }

src/Markdig/Helpers/StringSlice.cs

@@ -221,6 +221,22 @@ namespace Markdig.Helpers
             return false;
         }
 
+        /// <summary>
+        /// Searches for the specified character within this slice.
+        /// </summary>
+        /// <returns>A value >= 0 if the character was found, otherwise &lt; 0</returns>
+        public int IndexOf(char c)
+        {
+            for (int i = Start; i <= End; i++)
+            {
+                if (Text[i] == c)
+                {
+                    return i;
+                }
+            }
+            return -1;
+        }
+
         /// <summary>
         /// Searches the specified text within this slice (matching lowercase).
         /// </summary>

src/Markdig/Markdown.cs

@@ -30,13 +30,14 @@ namespace Markdig
         }
 
         /// <summary>
-        /// Converts a Markdown string to HTML.
+        /// Converts a Markdown string to HTML and output to the specified writer.
         /// </summary>
         /// <param name="markdown">A Markdown text.</param>
         /// <param name="writer">The destination <see cref="TextWriter"/> that will receive the result of the conversion.</param>
         /// <param name="pipeline">The pipeline used for the conversion.</param>
+        /// <returns>The Markdown document that has been parsed</returns>
         /// <exception cref="System.ArgumentNullException">if reader or writer variable are null</exception>
-        public static void ToHtml(string markdown, TextWriter writer, MarkdownPipeline pipeline = null)
+        public static MarkdownDocument ToHtml(string markdown, TextWriter writer, MarkdownPipeline pipeline = null)
         {
             if (markdown == null) throw new ArgumentNullException(nameof(markdown));
             if (writer == null) throw new ArgumentNullException(nameof(writer));
@@ -49,6 +50,8 @@ namespace Markdig
             var document = Parse(markdown, pipeline);
             renderer.Render(document);
             writer.Flush();
+
+            return document;
         }
 
         /// <summary>

src/Markdig/MarkdownPipeline.cs

@@ -47,7 +47,11 @@ namespace Markdig
 
         internal ProcessDocumentDelegate DocumentProcessed;
 
-        internal void Setup(IMarkdownRenderer renderer)
+        /// <summary>
+        /// Allows to setup a <see cref="IMarkdownRenderer"/>.
+        /// </summary>
+        /// <param name="renderer">The markdown renderer to setup</param>
+        public void Setup(IMarkdownRenderer renderer)
         {
             if (renderer == null) throw new ArgumentNullException(nameof(renderer));
             foreach (var extension in Extensions)

src/Markdig/Parsers/FencedBlockParserBase.cs

@@ -216,7 +216,7 @@ namespace Markdig.Parsers
             // The line must contain only fence opening character followed only by whitespaces.
             if (count <=0 && !processor.IsCodeIndent && (c == '\0' || c.IsWhitespace()) && line.TrimEnd())
             {
-                block.Span.End = line.Start - 1;
+                block.UpdateSpanEnd(line.Start - 1);
 
                 // Don't keep the last line
                 return BlockState.BreakDiscard;

src/Markdig/Parsers/HtmlBlockParser.cs

@@ -185,35 +185,35 @@ namespace Markdig.Parsers
                 case HtmlBlockType.Comment:
                     if (line.Search("-->", out endof))
                     {
-                        htmlBlock.Span.End = endof - 1;
+                        htmlBlock.UpdateSpanEnd(endof - 1);
                         result = BlockState.Break;
                     }
                     break;
                 case HtmlBlockType.CData:
                     if (line.Search("]]>", out endof))
                     {
-                        htmlBlock.Span.End = endof - 1;
+                        htmlBlock.UpdateSpanEnd(endof - 1);
                         result = BlockState.Break;
                     }
                     break;
                 case HtmlBlockType.ProcessingInstruction:
                     if (line.Search("?>", out endof))
                     {
-                        htmlBlock.Span.End = endof - 1;
+                        htmlBlock.UpdateSpanEnd(endof - 1);
                         result = BlockState.Break;
                     }
                     break;
                 case HtmlBlockType.DocumentType:
                     if (line.Search(">", out endof))
                     {
-                        htmlBlock.Span.End = endof - 1;
+                        htmlBlock.UpdateSpanEnd(endof - 1);
                         result = BlockState.Break;
                     }
                     break;
                 case HtmlBlockType.ScriptPreOrStyle:
                     if (line.SearchLowercase("</script>", out endof) || line.SearchLowercase("</pre>", out endof) || line.SearchLowercase("</style>", out endof))
                     {
-                        htmlBlock.Span.End = endof - 1;
+                        htmlBlock.UpdateSpanEnd(endof - 1);
                         result = BlockState.Break;
                     }
                     break;

src/Markdig/Parsers/IndentedCodeBlockParser.cs

@@ -48,7 +48,7 @@ namespace Markdig.Parsers
             }
             if (block != null)
             {
-                block.Span.End = processor.Line.End;
+                block.UpdateSpanEnd(processor.Line.End);
             }
             return BlockState.Continue;
         }

src/Markdig/Parsers/ListBlockParser.cs

@@ -153,7 +153,7 @@ namespace Markdig.Parsers
                 }
 
                 // Update list-item source end position
-                listItem.Span.End = state.Line.End;
+                listItem.UpdateSpanEnd(state.Line.End);
                 
                 return BlockState.Continue;
             }
@@ -170,11 +170,11 @@ namespace Markdig.Parsers
             {
                 if (state.Indent > columWidth && state.IsCodeIndent)
                 {
-                    state.GoToColumn(columWidth);
+                    state.GoToColumn(state.ColumnBeforeIndent + columWidth);
                 }
 
                 // Update list-item source end position
-                listItem.Span.End = state.Line.End;
+                listItem.UpdateSpanEnd(state.Line.End);
 
                 return BlockState.Continue;
             }
@@ -352,11 +352,11 @@ namespace Markdig.Parsers
                 isLastListItem = false;
             }
 
-            // Update end-position for the list
-            if (listBlock.Count > 0)
-            {
-                listBlock.Span.End = listBlock[listBlock.Count - 1].Span.End;
-            }
+            //// Update end-position for the list
+            //if (listBlock.Count > 0)
+            //{
+            //    listBlock.Span.End = listBlock[listBlock.Count - 1].Span.End;
+            //}
 
             return true;
         }

src/Markdig/Parsers/ParagraphBlockParser.cs

@@ -40,7 +40,7 @@ namespace Markdig.Parsers
                 return TryParseSetexHeading(processor, block);
             }
 
-            block.Span.End = processor.Line.End;
+            block.UpdateSpanEnd(processor.Line.End);
             return BlockState.Continue;
         }
 
@@ -139,7 +139,7 @@ namespace Markdig.Parsers
                 return BlockState.BreakDiscard;
             }
 
-            block.Span.End = state.Line.End;
+            block.UpdateSpanEnd(state.Line.End);
 
             return BlockState.Continue;
         }

src/Markdig/Parsers/QuoteBlockParser.cs

@@ -61,7 +61,6 @@ namespace Markdig.Parsers
             var c = processor.CurrentChar;
             if (c != quote.QuoteChar)
             {
-                block.Span.End = processor.Start - 1;
                 return processor.IsBlankLine ? BlockState.BreakDiscard : BlockState.None;
             }
 
@@ -71,18 +70,8 @@ namespace Markdig.Parsers
                 processor.NextChar(); // Skip following space
             }
 
-            block.Span.End = processor.Line.End;
+            block.UpdateSpanEnd(processor.Line.End);
             return BlockState.Continue;
         }
-
-        public override bool Close(BlockProcessor processor, Block block)
-        {
-            var quoteBlock = block as QuoteBlock;
-            if (quoteBlock?.LastChild != null)
-            {
-                quoteBlock.Span.End = quoteBlock.LastChild.Span.End;
-            }
-            return true;
-        }
     }
 }

src/Markdig/Properties/AssemblyInfo.cs

@@ -25,6 +25,6 @@ namespace Markdig
 {
     public static partial class Markdown
     {
-        public const string Version = "0.5.5";
+        public const string Version = "0.5.9";
     }
 }

src/Markdig/Syntax/Block.cs

@@ -75,5 +75,19 @@ namespace Markdig.Syntax
         {
             ProcessInlinesEnd?.Invoke(state, null);
         }
+
+        public void UpdateSpanEnd(int spanEnd)
+        {
+            // Update parent spans
+            var parent = this;
+            while (parent != null)
+            {
+                if (spanEnd > parent.Span.End)
+                {
+                    parent.Span.End = spanEnd;
+                }
+                parent = parent.Parent;
+            }
+        }
     }
 }

src/Markdig/Syntax/BlockExtensions.cs

@@ -0,0 +1,138 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license. 
+// See the license.txt file in the project root for more information.
+namespace Markdig.Syntax
+{
+    /// <summary>
+    /// Extensions for <see cref="Block"/>
+    /// </summary>
+    public static class BlockExtensions
+    {
+        // TODO: Add test for this code
+
+        public static Block FindBlockAtPosition(this Block rootBlock, int position)
+        {
+            var contains = rootBlock.CompareToPosition(position) == 0;
+            var blocks = rootBlock as ContainerBlock;
+            if (blocks == null || blocks.Count == 0 || !contains)
+            {
+                return contains ? rootBlock : null;
+            }
+
+            var lowerIndex = 0;
+            var upperIndex = blocks.Count - 1;
+
+            // binary search on lines
+            Block block = null;
+            while (lowerIndex <= upperIndex)
+            {
+                int midIndex = (upperIndex - lowerIndex) / 2 + lowerIndex;
+                block = blocks[midIndex];
+                int comparison = block.CompareToPosition(position);
+                if (comparison == 0)
+                {
+                    break;
+                }
+
+                block = null;
+                if (comparison < 0)
+                    lowerIndex = midIndex + 1;
+                else
+                    upperIndex = midIndex - 1;
+            }
+
+            if (block == null)
+            {
+                return rootBlock;
+            }
+
+            // Recursively go deep into the block
+            return FindBlockAtPosition(block, position);
+        }
+
+
+        public static int FindClosestLine(this MarkdownDocument root, int line)
+        {
+            var closestBlock = root.FindClosestBlock(line);
+            return closestBlock?.Line ?? 0;
+        }
+
+        public static Block FindClosestBlock(this Block rootBlock, int line)
+        {
+            var blocks = rootBlock as ContainerBlock;
+            if (blocks == null || blocks.Count == 0)
+            {
+                return rootBlock.Line == line ? rootBlock : null;
+            }
+
+            var lowerIndex = 0;
+            var upperIndex = blocks.Count - 1;
+
+            // binary search on lines
+            while (lowerIndex <= upperIndex)
+            {
+                int midIndex = (upperIndex - lowerIndex) / 2 + lowerIndex;
+                var block = blocks[midIndex];
+                int comparison = block.Line.CompareTo(line);
+                if (comparison == 0)
+                {
+                    return block;
+                }
+                if (comparison < 0)
+                    lowerIndex = midIndex + 1;
+                else
+                    upperIndex = midIndex - 1;
+            }
+
+            // If we are between two lines, try to find the best spot
+            if (lowerIndex > 0 && lowerIndex < blocks.Count)
+            {
+                var prevBlock = blocks[lowerIndex - 1].FindClosestBlock(line) ?? blocks[lowerIndex - 1];
+                var nextBlock = blocks[lowerIndex].FindClosestBlock(line) ?? blocks[lowerIndex];
+
+                if (prevBlock.Line == line)
+                {
+                    return prevBlock;
+                }
+
+                if (nextBlock.Line == line)
+                {
+                    return nextBlock;
+                }
+
+                // we calculate the position of the current line relative to the line found and previous line
+                var prevLine = prevBlock.Line;
+                var nextLine = nextBlock.Line;
+
+                var middle = (line - prevLine) * 1.0 / (nextLine - prevLine);
+                // If  relative position < 0.5, we select the previous line, otherwise we select the line found
+                return middle < 0.5 ? prevBlock : nextBlock;
+            }
+
+            if (lowerIndex == 0)
+            {
+                var prevBlock = blocks[lowerIndex].FindClosestBlock(line) ?? blocks[lowerIndex];
+                return prevBlock;
+            }
+
+            if (lowerIndex == blocks.Count)
+            {
+                var prevBlock = blocks[lowerIndex - 1].FindClosestBlock(line) ?? blocks[lowerIndex - 1];
+                return prevBlock;
+            }
+
+            return null;
+        }
+
+
+        public static bool ContainsPosition(this Block block, int position)
+        {
+            return CompareToPosition(block, position) == 0;
+        }
+
+        public static int CompareToPosition(this Block block, int position)
+        {
+            return position < block.Span.Start ? 1 : position > block.Span.End + 1 ? -1 : 0;
+        }
+    }
+}

src/Markdig/Syntax/ContainerBlock.cs

@@ -69,10 +69,7 @@ namespace Markdig.Syntax
             children[Count++] = item;
             item.Parent = this;
 
-            if (item.Span.End > Span.End)
-            {
-                Span.End = item.Span.End;
-            }
+            UpdateSpanEnd(item.Span.End);
         }
 
         private void EnsureCapacity(int min)

src/Markdig/project.json

@@ -1,6 +1,6 @@
 {
   "title": "Markdig",
-  "version": "0.5.5",
+  "version": "0.5.9",
   "authors": [ "Alexandre Mutel" ],
   "description": "A fast, powerfull, CommonMark compliant, extensible Markdown processor for .NET",
   "copyright": "Alexandre Mutel",
@@ -11,7 +11,7 @@
     "projectUrl": "https://github.com/lunet-io/markdig",
     "iconUrl": "https://raw.githubusercontent.com/lunet-io/markdig/master/img/markdig.png",
     "requireLicenseAcceptance": false,
-    "releaseNotes": "> 0.5.5\n- Add same github class for task lists\n- Add pragma lines extension\n> 0.5.4:\nFix bug in HTML block parsing which could break parsing of remaining document",
+    "releaseNotes": "- Fix ArgumentOutOfRangeException for FindClosestBlock\n- Fix bug in nested list items\n",
     "tags": [ "Markdown CommonMark md html md2html" ]
   },
   "configurations": {