Merge pull request #736 from zickb/better_literal_delimiter_content_string_slice

Better literal delimiter content string slice

src/Markdig.Tests/MiscTests.cs

@@ -1,7 +1,8 @@
 using System.Text.RegularExpressions;
 
 using Markdig.Extensions.AutoLinks;
-
+using Markdig.Extensions.Tables;
+using Markdig.Syntax;
 using NUnit.Framework;
 
 namespace Markdig.Tests;
@@ -198,9 +199,9 @@ $$
 <div class=""math"">
 \begin{align}
 \sqrt{37} & = \sqrt{\frac{73^2-1}{12^2}} \\
- & = \sqrt{\frac{73^2}{12^2}\cdot\frac{73^2-1}{73^2}} \\ 
+ & = \sqrt{\frac{73^2}{12^2}\cdot\frac{73^2-1}{73^2}} \\
  & = \sqrt{\frac{73^2}{12^2}}\sqrt{\frac{73^2-1}{73^2}} \\
- & = \frac{73}{12}\sqrt{1 - \frac{1}{73^2}} \\ 
+ & = \frac{73}{12}\sqrt{1 - \frac{1}{73^2}} \\
  & \approx \frac{73}{12}\left(1 - \frac{1}{2\cdot73^2}\right)
 \end{align}
 </div>
@@ -291,4 +292,29 @@ $$
         TestParser.TestSpec("www.foo.bar", "<p><a href=\"http://www.foo.bar\">www.foo.bar</a></p>", pipeline);
         TestParser.TestSpec("www.foo.bar", "<p><a href=\"https://www.foo.bar\">www.foo.bar</a></p>", httpsPipeline);
     }
+
+    [Test]
+    public void RootInlineHasCorrectSourceSpan()
+    {
+        var pipeline = new MarkdownPipelineBuilder().UsePreciseSourceLocation().Build();
+        pipeline.TrackTrivia = true;
+
+        var document = Markdown.Parse("0123456789\n", pipeline);
+
+        var expectedSourceSpan = new SourceSpan(0, 10);
+        Assert.That(((LeafBlock)document.LastChild).Inline.Span == expectedSourceSpan);
+    }
+
+    [Test]
+    public void RootInlineInTableCellHasCorrectSourceSpan()
+    {
+        var pipeline = new MarkdownPipelineBuilder().UsePreciseSourceLocation().UseAdvancedExtensions().Build();
+        pipeline.TrackTrivia = true;
+
+        var document = Markdown.Parse("| a | b |\n| --- | --- |\n| <span id=\"dest\"></span><span id=\"DEST\"></span>*dest*<br/> | \\[in\\] The address of the result of the operation.<br/> |", pipeline);
+
+        var paragraph = (ParagraphBlock)((TableCell)((TableRow)((Table)document.LastChild).LastChild).First()).LastChild;
+        Assert.That(paragraph.Inline.Span.Start == paragraph.Inline.FirstChild.Span.Start);
+        Assert.That(paragraph.Inline.Span.End == paragraph.Inline.LastChild.Span.End);
+    }
 }

src/Markdig.Tests/TestEmphasisPlus.cs

@@ -1,7 +1,10 @@
 // Copyright (c) Alexandre Mutel. All rights reserved.
-// This file is licensed under the BSD-Clause 2 license. 
+// This file is licensed under the BSD-Clause 2 license.
 // See the license.txt file in the project root for more information.
 
+using Markdig.Syntax;
+using Markdig.Syntax.Inlines;
+
 namespace Markdig.Tests;
 
 [TestFixture]
@@ -18,4 +21,17 @@ public partial class TestEmphasisPlus
     {
         TestParser.TestSpec("normal ***Strong emphasis*** normal", "<p>normal <em><strong>Strong emphasis</strong></em> normal</p>", "");
     }
+
+    [Test]
+    public void OpenEmphasisHasConvenientContentStringSlice()
+    {
+        var pipeline = new MarkdownPipelineBuilder().Build();
+
+        var document = Markdown.Parse("test*test", pipeline);
+
+        var emphasisDelimiterLiteral = (LiteralInline)((ParagraphBlock)document.LastChild).Inline.ElementAt(1);
+        Assert.That(emphasisDelimiterLiteral.Content.Text == "test*test");
+        Assert.That(emphasisDelimiterLiteral.Content.Start == 4);
+        Assert.That(emphasisDelimiterLiteral.Content.End == 4);
+    }
 }

src/Markdig.Tests/TestMediaLinks.cs

@@ -36,6 +36,16 @@ public class TestMediaLinks
         Assert.AreEqual(html, expected);
     }
 
+    [TestCase("![static video relative path](./video.mp4)",
+        "<p><video width=\"500\" height=\"281\" controls=\"\"><source type=\"video/mp4\" src=\"./video.mp4\"></source></video></p>\n")]
+    [TestCase("![static audio relative path](./audio.mp3)",
+        "<p><audio width=\"500\" controls=\"\"><source type=\"audio/mpeg\" src=\"./audio.mp3\"></source></audio></p>\n")]
+    public void TestBuiltInHostsWithRelativePaths(string markdown, string expected)
+    {
+        string html = Markdown.ToHtml(markdown, GetPipeline());
+        Assert.AreEqual(html, expected);
+    }
+    
     private class TestHostProvider : IHostProvider
     {
         public string Class { get; } = "regex";

src/Markdig.Tests/TestSourcePosition.cs

@@ -67,6 +67,28 @@ literal      ( 2, 0) 12-21
 ");
     }
 
+    [Test]
+    public void TestParagraphWithEndNewLine()
+    {
+        Check("0123456789\n", @"
+paragraph    ( 0, 0)  0-10
+literal      ( 0, 0)  0-9
+linebreak    ( 0,10) 10-10
+", trackTrivia: true);
+
+        Check("0123456789\r", @"
+paragraph    ( 0, 0)  0-10
+literal      ( 0, 0)  0-9
+linebreak    ( 0,10) 10-10
+", trackTrivia: true);
+
+        Check("0123456789\r\n", @"
+paragraph    ( 0, 0)  0-11
+literal      ( 0, 0)  0-9
+linebreak    ( 0,10) 10-11
+", trackTrivia: true);
+    }
+
     [Test]
     public void TestEmphasis()
     {
@@ -825,9 +847,10 @@ literal      ( 8, 2) 77-92
 ");
     }
 
-    private static void Check(string text, string expectedResult, string extensions = null)
+    private static void Check(string text, string expectedResult, string extensions = null, bool trackTrivia = false)
     {
         var pipelineBuilder = new MarkdownPipelineBuilder().UsePreciseSourceLocation();
+        pipelineBuilder.TrackTrivia = trackTrivia;
         if (extensions != null)
         {
             pipelineBuilder.Configure(extensions);

src/Markdig.Tests/TestStringSliceList.cs

@@ -180,4 +180,39 @@ public class TestStringSliceList
             Assert.AreEqual('\0', iterator.CurrentChar); iterator.SkipChar();
         }
     }
+
+    [Test]
+    public void TestStringLineGroupCharIteratorAtCapacity()
+    {
+        string str = "ABCDEFGHI";
+        var text = new StringLineGroup(1)
+        {
+            // Will store the following line at capacity
+            new StringSlice(str, NewLine.CarriageReturnLineFeed) { Start = 0, End = 2 },
+        };
+
+        var iterator = text.ToCharIterator();
+        var chars = ToString(iterator);
+        TextAssert.AreEqual("ABC\r\n", chars.ToString());
+        TextAssert.AreEqual("ABC", text.ToString());
+    }
+
+    [Test]
+    public void TestStringLineGroupCharIteratorForcingIncreaseCapacity()
+    {
+        string str = "ABCDEFGHI";
+        var text = new StringLineGroup(1)
+        {
+            // Will store the following line at capacity
+            new StringSlice(str, NewLine.CarriageReturnLineFeed) { Start = 0, End = 2 },
+
+            // Will force increase capacity to 2 and store the line at capacity
+            new StringSlice(str, NewLine.CarriageReturnLineFeed) { Start = 3, End = 3 },
+        };
+
+        var iterator = text.ToCharIterator();
+        var chars = ToString(iterator);
+        TextAssert.AreEqual("ABC\r\nD\r\n", chars.ToString());
+        TextAssert.AreEqual("ABC\r\nD", text.ToString());
+    }
 }

src/Markdig.Tests/TestYamlFrontMatterExtension.cs

@@ -83,4 +83,28 @@ public class TestYamlFrontMatterExtension
             return null;
         }
     }
+
+    [TestCase("---\nkey1: value1\nkey2: value2\n---\n\n# Content\n")]
+    [TestCase("---\nkey1: value1\nkey2: value2\nkey3: value3\nkey4: value4\nkey5: value5\nkey6: value6\nkey7: value7\nkey8: value8\n---\n\n# Content\n")]
+    public void FrontMatterBlockLinesCharIterator(string value)
+    {
+        var builder = new MarkdownPipelineBuilder();
+        builder.Extensions.Add(new YamlFrontMatterExtension());
+        var markdownDocument = Markdown.Parse(value, builder.Build());
+
+        var yamlBlocks = markdownDocument.Descendants<YamlFrontMatterBlock>();
+        Assert.True(yamlBlocks.Any());
+
+        foreach (var yamlBlock in yamlBlocks)
+        {
+            var iterator = yamlBlock.Lines.ToCharIterator();
+            while(iterator.CurrentChar != '\0')
+            {
+                iterator.NextChar();
+            }
+        }
+
+        Assert.Pass("No exception parsing and iterating through YAML front matter block lines");
+    }
+
 }

src/Markdig/Extensions/MediaLinks/MediaLinkExtension.cs

@@ -50,27 +50,32 @@ public class MediaLinkExtension : IMarkdownExtension
             return false;
         }
 
+        var url = linkInline.Url;
         bool isSchemaRelative = false;
-        // Only process absolute Uri
-        if (!Uri.TryCreate(linkInline.Url, UriKind.RelativeOrAbsolute, out Uri? uri) || !uri.IsAbsoluteUri)
+
+        // force // schema to an absolute url
+        if (url.StartsWith("//", StringComparison.Ordinal))
         {
-            // see https://tools.ietf.org/html/rfc3986#section-4.2
-            // since relative uri doesn't support many properties, "http" is used as a placeholder here.
-            if (linkInline.Url.StartsWith("//", StringComparison.Ordinal) && Uri.TryCreate("http:" + linkInline.Url, UriKind.Absolute, out uri))
+            url = "https:" + url;
+            isSchemaRelative = true;
+        }
+
+        // Make sure we have a valid absolute/relative url
+        if (!Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out Uri? uri)) // || !uri.IsAbsoluteUri)
+        {
+            return false;
+        }
+        
+        // iFrame has to be absolute path
+        if (uri.IsAbsoluteUri)
+        {
+            if (TryRenderIframeFromKnownProviders(uri, isSchemaRelative, renderer, linkInline))
             {
-                isSchemaRelative = true;
-            }
-            else
-            {
-                return false;
+                return true;
             }
         }
 
-        if (TryRenderIframeFromKnownProviders(uri, isSchemaRelative, renderer, linkInline))
-        {
-            return true;
-        }
-
+        // audio/video has can have relative path
         if (TryGuessAudioVideoFile(uri, isSchemaRelative, renderer, linkInline))
         {
             return true;
@@ -93,7 +98,10 @@ public class MediaLinkExtension : IMarkdownExtension
 
     private bool TryGuessAudioVideoFile(Uri uri, bool isSchemaRelative, HtmlRenderer renderer, LinkInline linkInline)
     {
-        var path = uri.GetComponents(UriComponents.Path, UriFormat.Unescaped);
+        string path = uri.IsAbsoluteUri
+            ? uri.GetComponents(UriComponents.Path, UriFormat.Unescaped)
+            : uri.ToString();
+        
         // Otherwise try to detect if we have an audio/video from the file extension
         var lastDot = path.LastIndexOf('.');
         if (lastDot >= 0 &&

src/Markdig/Extensions/Tables/PipeTableParser.cs

@@ -443,6 +443,11 @@ public class PipeTableParser : InlineParser, IPostInlineProcessor
         {
             var paragraph = (ParagraphBlock) cell[0];
             state.PostProcessInlines(postInlineProcessorIndex + 1, paragraph.Inline, null, true);
+            if (paragraph.Inline?.LastChild is not null)
+            {
+                paragraph.Inline.Span.End = paragraph.Inline.LastChild.Span.End;
+                paragraph.UpdateSpanEnd(paragraph.Inline.LastChild.Span.End);
+            }
         }
 
         // Clear cells when we are done
@@ -520,7 +525,7 @@ public class PipeTableParser : InlineParser, IPostInlineProcessor
                 // Create aligns until we may have a header row
 
                 aligns ??= new List<TableColumnDefinition>();
-                
+
                 aligns.Add(new TableColumnDefinition() { Alignment =  align });
 
                 // If this is the last delimiter, we need to check the right side of the `|` delimiter

src/Markdig/Helpers/StringLineGroup.cs

@@ -334,9 +334,12 @@ public struct StringLineGroup : IEnumerable
             goto Return;
 
         MoveToNewLine:
-            SliceIndex++;
-            _offset = -1;
-            _currentSlice = _lines.Lines[SliceIndex];
+            if (SliceIndex < _lines.Lines.Length - 1)
+            {
+                SliceIndex++;
+                _offset = -1;
+                _currentSlice = _lines.Lines[SliceIndex];
+            }
 
         Return:
             return CurrentChar;

src/Markdig/Markdown.cs

@@ -17,7 +17,7 @@ namespace Markdig;
 /// <summary>
 /// Provides methods for parsing a Markdown string to a syntax tree and converting it to other formats.
 /// </summary>
-public static partial class Markdown
+public static class Markdown
 {
     public static string Version
     {
@@ -268,7 +268,7 @@ public static partial class Markdown
     }
 
     /// <summary>
-    /// Converts a Markdown string to HTML.
+    /// Converts a Markdown string to Plain text by using a <see cref="StringWriter"/> .
     /// </summary>
     /// <param name="markdown">A Markdown text.</param>
     /// <param name="pipeline">The pipeline used for the conversion.</param>

src/Markdig/Parsers/InlineProcessor.cs

@@ -225,6 +225,7 @@ public class InlineProcessor
         previousLineIndexForSliceOffset = 0;
         lineOffsets.Clear();
         var text = leafBlock.Lines.ToSlice(lineOffsets);
+        var textEnd = text.Length;
         leafBlock.Lines.Release();
         int previousStart = -1;
 
@@ -319,7 +320,8 @@ public class InlineProcessor
                 var newLine = leafBlock.NewLine;
                 if (newLine != NewLine.None)
                 {
-                    leafBlock.Inline.AppendChild(new LineBreakInline { NewLine = newLine });
+                    var position = GetSourcePosition(textEnd, out int line, out int column);
+                    leafBlock.Inline.AppendChild(new LineBreakInline { NewLine = newLine, Line = line, Column = column, Span = { Start = position, End = position + (newLine == NewLine.CarriageReturnLineFeed ? 1 : 0) } });
                 }
             }
         }
@@ -342,6 +344,12 @@ public class InlineProcessor
         //    DebugLog.WriteLine("** Dump after Emphasis:");
         //    leafBlock.Inline.DumpTo(DebugLog);
         //}
+
+        if (leafBlock.Inline.LastChild is not null)
+        {
+            leafBlock.Inline.Span.End = leafBlock.Inline.LastChild.Span.End;
+            leafBlock.UpdateSpanEnd(leafBlock.Inline.Span.End);
+        }
     }
 
     public void PostProcessInlines(int startingIndex, Inline? root, Inline? lastChild, bool isFinalProcessing)

src/Markdig/Parsers/Inlines/EmphasisInlineParser.cs

@@ -1,5 +1,5 @@
 // Copyright (c) Alexandre Mutel. All rights reserved.
-// This file is licensed under the BSD-Clause 2 license. 
+// This file is licensed under the BSD-Clause 2 license.
 // See the license.txt file in the project root for more information.
 
 using System.Diagnostics;
@@ -109,7 +109,7 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
         var child = container.FirstChild;
         while (child != null)
         {
-            // Stop the search on the delimitation child 
+            // Stop the search on the delimitation child
             if (child == lastChild)
             {
                 break;
@@ -197,7 +197,7 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
             if (canOpen) delimiterType |= DelimiterType.Open;
             if (canClose) delimiterType |= DelimiterType.Close;
 
-            var delimiter = new EmphasisDelimiterInline(this, emphasisDesc)
+            var delimiter = new EmphasisDelimiterInline(this, emphasisDesc, new StringSlice(slice.Text, startPosition, slice.Start - 1))
             {
                 DelimiterCount = delimiterCount,
                 Type = delimiterType,
@@ -221,7 +221,7 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
 
         // TODO: Benchmark difference between using List and LinkedList here since there could be a few Remove calls
 
-        // Move current_position forward in the delimiter stack (if needed) until 
+        // Move current_position forward in the delimiter stack (if needed) until
         // we find the first potential closer with delimiter * or _. (This will be the potential closer closest to the beginning of the input – the first one in parse order.)
         for (int i = 0; i < delimiters.Count; i++)
         {
@@ -237,7 +237,7 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
             {
                 while (true)
                 {
-                    // Now, look back in the stack (staying above stack_bottom and the openers_bottom for this delimiter type) 
+                    // Now, look back in the stack (staying above stack_bottom and the openers_bottom for this delimiter type)
                     // for the first matching potential opener (“matching” means same delimiter).
                     EmphasisDelimiterInline? openDelimiter = null;
                     int openDelimiterIndex = -1;
@@ -307,8 +307,10 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
                         emphasis.Column = openDelimiter.Column;
                         emphasis.Span.End = closeDelimiter.Span.End - closeDelimitercount + delimiterDelta;
 
+                        openDelimiter.Content.Start += delimiterDelta;
                         openDelimiter.Span.Start += delimiterDelta;
                         openDelimiter.Column += delimiterDelta;
+                        closeDelimiter.Content.Start += delimiterDelta;
                         closeDelimiter.Span.Start += delimiterDelta;
                         closeDelimiter.Column += delimiterDelta;
 
@@ -331,7 +333,7 @@ public class EmphasisInlineParser : InlineParser, IPostInlineProcessor
                         for (int k = i - 1; k >= openDelimiterIndex + 1; k--)
                         {
                             var literalDelimiter = delimiters[k];
-                            literalDelimiter.ReplaceBy(literalDelimiter.AsLiteralInline());                                
+                            literalDelimiter.ReplaceBy(literalDelimiter.AsLiteralInline());
                             delimiters.RemoveAt(k);
                             i--;
                         }

src/Markdig/Polyfills/NullableAttributes.cs

@@ -17,7 +17,7 @@ internal sealed class NotNullWhenAttribute : Attribute
 }
 
 [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, Inherited = false)]
-public sealed class AllowNullAttribute : Attribute { }
+internal sealed class AllowNullAttribute : Attribute { }
 #endif
 
 #if !NET5_0_OR_GREATER

src/Markdig/Syntax/Inlines/EmphasisDelimiterInline.cs

@@ -1,5 +1,5 @@
 // Copyright (c) Alexandre Mutel. All rights reserved.
-// This file is licensed under the BSD-Clause 2 license. 
+// This file is licensed under the BSD-Clause 2 license.
 // See the license.txt file in the project root for more information.
 
 using Markdig.Helpers;
@@ -27,6 +27,24 @@ public class EmphasisDelimiterInline : DelimiterInline
 
         Descriptor = descriptor;
         DelimiterChar = descriptor.Character;
+        Content = new StringSlice(ToLiteral());
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="EmphasisDelimiterInline" /> class.
+    /// </summary>
+    /// <param name="parser">The parser.</param>
+    /// <param name="descriptor">The descriptor.</param>
+    /// <param name="content">The content.</param>
+    /// <exception cref="ArgumentNullException"></exception>
+    internal EmphasisDelimiterInline(InlineParser parser, EmphasisDescriptor descriptor, StringSlice content) : base(parser)
+    {
+        if (descriptor is null)
+            ThrowHelper.ArgumentNullException(nameof(descriptor));
+
+        Descriptor = descriptor;
+        DelimiterChar = descriptor.Character;
+        Content = content;
     }
 
     /// <summary>
@@ -44,6 +62,11 @@ public class EmphasisDelimiterInline : DelimiterInline
     /// </summary>
     public int DelimiterCount { get; set; }
 
+    /// <summary>
+    /// The content as a <see cref="StringSlice"/>.
+    /// </summary>
+    public StringSlice Content;
+
     public override string ToLiteral()
     {
         return DelimiterCount > 0 ? new string(DelimiterChar, DelimiterCount) : string.Empty;
@@ -53,7 +76,7 @@ public class EmphasisDelimiterInline : DelimiterInline
     {
         return new LiteralInline()
         {
-            Content = new StringSlice(ToLiteral()),
+            Content = Content,
             IsClosed = true,
             Span = Span,
             Line = Line,

src/global.json

@@ -1,7 +1,7 @@
 {
   "sdk": {
     "version": "6.0.100",
-    "rollForward": "latestMinor",
+    "rollForward": "latestMajor",
     "allowPrerelease": false
   }
 }