feat: Use core.titleFromHeading in parsing and semantic analysis (#369)

The setting is now used during parsing
CST -> AST -> Syms transformation can now use 'isTitle' metadata on a
heading node.

Author: artempyanykh

Marksman/Ast.fs

@@ -6,6 +6,7 @@ open Marksman.Syms
 
 type Heading = {
     level: int
+    isTitle: bool
     text: string
     id: Slug
 } with
@@ -111,14 +112,13 @@ module Element =
     // TODO: instead of checking for 'all whitespace' symbols all the time, create smart constructors
     let toSym (parserSettings: Config.ParserSettings) (el: Element) : option<Sym> =
         match el with
-        | Element.H { level = level; id = id } ->
+        | Element.H { level = level; isTitle = isTitle; id = id } ->
             if Slug.isEmpty id then
                 None
             else
                 let id = Slug.toString id
 
-                // TODO: make this configurable
-                if level <= 1 then
+                if isTitle then
                     Syms.Sym.Def(Title(id)) |> Some
                 else
                     Syms.Sym.Def(Header(level, id)) |> Some

Marksman/Cst.fs

@@ -252,7 +252,7 @@ type Element =
         | T n -> n.range
         | YML n -> n.range
 
-and Heading = { level: int; title: TextNode; scope: Range }
+and Heading = { level: int; isTitle: bool; title: TextNode; scope: Range }
 
 let rec private fmtElement =
     function
@@ -298,14 +298,15 @@ module Heading =
 
     let slug (heading: Heading) : Slug = name heading |> Slug.ofString
 
-    let isTitle (heading: Heading) = heading.level <= 1
+    let isTitle (heading: Heading) = heading.isTitle
 
     let range (heading: Heading) : Range = heading.title.range
 
     let scope (heading: Heading) : Range = heading.scope
 
     let toAbstract (cHead: Heading) : Ast.Heading = {
         level = cHead.level
+        isTitle = cHead.isTitle
         text = cHead.title.text
         id = slug cHead
     }

Marksman/Index.fs

@@ -7,6 +7,7 @@ open Marksman.Misc
 
 type Dictionary<'K, 'V> = System.Collections.Generic.Dictionary<'K, 'V>
 
+// TODO: get rid of this; use Structure directly
 type Index = {
     titles: array<Node<Heading>>
     headings: array<Node<Heading>>

Marksman/Parser.fs

@@ -5,6 +5,7 @@ open System.Collections.Generic
 open Ionide.LanguageServerProtocol.Types
 open Markdig.Syntax
 
+open Marksman.Config
 open Marksman.Misc
 open Marksman.Names
 open Marksman.Text
@@ -216,7 +217,7 @@ module Markdown =
             }
 
 
-    let scrapeText (text: Text) : array<Element> =
+    let scrapeText (parserSettings: ParserSettings) (text: Text) : array<Element> =
         let parsed: MarkdownObject = Markdown.Parse(text.content, markdigPipeline)
 
         let elements = ResizeArray()
@@ -250,6 +251,7 @@ module Markdown =
                 let heading =
                     Node.mk fullText range {
                         level = level
+                        isTitle = parserSettings.titleFromHeading && level <= 1
                         title = Node.mkText title titleRange
                         scope = range
                     }
@@ -476,11 +478,11 @@ module Markdown =
 
         { elements = elements; childMap = childMap }
 
-let parse (parserSettings: Config.ParserSettings) (text: Text) : Structure =
+let parse (parserSettings: ParserSettings) (text: Text) : Structure =
     if String.IsNullOrEmpty text.content then
         let cst: Cst.Cst = { elements = [||]; childMap = Map.empty }
         Structure.ofCst parserSettings cst
     else
-        let flatElements = Markdown.scrapeText text
+        let flatElements = Markdown.scrapeText parserSettings text
         let cst = Markdown.buildCst text flatElements
         Structure.ofCst parserSettings cst

Tests/AstTests.fs

@@ -1,5 +1,6 @@
 module Marksman.AstTests
 
+open Marksman.Config
 open Xunit
 open Snapper
 
@@ -62,3 +63,51 @@ let testAstLookup () =
 
     let madeUpAbstract = Element.MR(Collapsed "WAT")
     Assert.Equal(Structure.tryFindConcreteForAbstract madeUpAbstract struct1, None)
+
+[<Fact>]
+let testSymsWhenTitleFromHeadingIsOff () =
+    let doc =
+        """
+# H1
+Is this a title?
+# H2
+Is this another title?
+## H2.2
+# H3
+And this?
+"""
+
+    let strukt =
+        Parser.parse { ParserSettings.Default with titleFromHeading = false } (Text.mkText doc)
+
+    Helpers.checkInlineSnapshot _.ToString() strukt.Symbols [
+        "Doc"
+        "H1 {h1}"
+        "H1 {h2}"
+        "H1 {h3}"
+        "H2 {h22}"
+    ]
+
+[<Fact>]
+let testSymsWhenTitleFromHeadingIsOn () =
+    let doc =
+        """
+# H1
+Is this a title?
+# H2
+Is this another title?
+## H2.2
+# H3
+And this?
+"""
+
+    let strukt =
+        Parser.parse { ParserSettings.Default with titleFromHeading = true } (Text.mkText doc)
+
+    Helpers.checkInlineSnapshot _.ToString() strukt.Symbols [
+        "Doc"
+        "T {h1}"
+        "T {h2}"
+        "T {h3}"
+        "H2 {h22}"
+    ]

Tests/ConnTest.fs

@@ -1,5 +1,6 @@
 module Marksman.ConnTest
 
+open Marksman.Config
 open Xunit
 
 open Snapper
@@ -363,3 +364,35 @@ module ConnGraphTests =
         let d2 = FakeDoc.Mk(path = "d2.md", contentLines = [| "#tag2"; "#tag3" |])
         let f' = mkFolder [ d1; d2 ] |> Folder.withDoc d1'
         checkSnapshot (Folder.conn f')
+
+[<StoreSnapshotsPerClass>]
+module ConnGraphTests_TitleLess =
+    let incrConfig = {
+        Config.Config.Default with
+            coreIncrementalReferences = Some true
+            coreTitleFromHeading = Some false
+            complWikiStyle = Some ComplWikiStyle.TitleSlug
+    }
+
+    let mkFolder docs = FakeFolder.Mk(config = incrConfig, docs = docs)
+    let mkDoc path content = FakeDoc.Mk(path = path, config = incrConfig, contentLines = content)
+
+    [<Fact>]
+    let updateH1 () =
+        let d1 =
+            mkDoc "ocaml.md" [| "# OCaml"; "[[#Multicore]]"; "## Multicore" |]
+
+        let d1' =
+            mkDoc "ocaml.md" [| "# OCaml L"; "[[#Multicore]]"; "## Multicore" |]
+
+        let d2 = mkDoc "test.md" [| "# Test"; "[[OCaml#Multicore]]" |]
+
+        let incr =
+            mkFolder [ d1 ]
+            |> Folder.withDoc d2
+            |> Folder.withDoc d1'
+            |> Folder.conn
+
+        let fromScratch = mkFolder [ d1'; d2 ] |> Folder.conn
+        let connDiff = Conn.difference fromScratch incr
+        checkInlineSnapshot id [ connDiff.CompactFormat() ] [ "" ]

Tests/Helpers.fs

@@ -1,6 +1,7 @@
 module Marksman.Helpers
 
 open System.Runtime.InteropServices
+open Marksman.Config
 open Snapper
 open Marksman.Misc
 open Marksman.Paths
@@ -53,21 +54,22 @@ let stripMarginTrim (str: string) = stripMargin (str.Trim())
 
 type FakeDoc =
     class
-        static member Mk(content: string, ?path: string, ?root: string) : Doc =
+        static member Mk(content: string, ?path: string, ?root: string, ?config: Config) : Doc =
             let text = Text.mkText content
             let path = defaultArg path "fake.md"
             let pathUri = pathToUri (dummyRootPath (pathComps path))
             let root = Option.map pathComps root |> Option.defaultValue []
             let rootUri = dummyRootPath root |> pathToUri
+            let config = defaultArg config Config.Default
 
             let docId =
                 DocId(UriWith.mkRooted (UriWith.mkRoot rootUri) (LocalPath.ofUri pathUri))
 
-            Doc.mk Config.ParserSettings.Default docId None text
+            Doc.mk (ParserSettings.OfConfig(config)) docId None text
 
-        static member Mk(contentLines: array<string>, ?path: string) : Doc =
+        static member Mk(contentLines: array<string>, ?path: string, ?config: Config) : Doc =
             let content = String.concat System.Environment.NewLine contentLines
-            FakeDoc.Mk(content, ?path = path)
+            FakeDoc.Mk(content, ?path = path, ?config = config)
     end
 
 type FakeFolder =

Tests/RefsTests.fs

@@ -2,6 +2,7 @@ module Marksman.RefsTests
 
 open Ionide.LanguageServerProtocol.Types
 open System.IO
+open Marksman.Config
 open Xunit
 
 open Marksman.Cst
@@ -574,6 +575,39 @@ module EncodingTests =
         let refs = resolveAtPos doc1 14 5
         checkInlineSnapshot (fun x -> x.ToString()) refs [ "doc.3.with.dots" ]
 
+// Cases where title_from_heading = false
+module TitleLess =
+    [<Fact>]
+    let refToH1 () =
+        let baseConfig = {
+            Config.Config.Default with
+                complWikiStyle = Some ComplWikiStyle.TitleSlug
+        }
+
+        let mkFolder config =
+            let d1 =
+                FakeDoc.Mk(path = "d1.md", config = config, contentLines = [| "# Doc1" |])
+
+            let d2 =
+                //                                                             012345
+                FakeDoc.Mk(path = "d2.md", config = config, contentLines = [| "[[Doc1]]" |])
+
+            let folder = FakeFolder.Mk([ d1; d2 ], config = config)
+            d1, d2, folder
+
+        // In title-less mode headings are not referenceable cross-doc
+        let _, d2, folder =
+            mkFolder { baseConfig with coreTitleFromHeading = Some false }
+
+        let el = requireElementAtPos d2 0 2
+        Assert.Empty(Dest.tryResolveElement folder d2 el)
+
+        // In title-full mode headings *are* referenceable cross-doc
+        let _, d2, folder =
+            mkFolder { baseConfig with coreTitleFromHeading = Some true }
+
+        Assert.NotEmpty(Dest.tryResolveElement folder d2 el)
+
 module RegressionTests =
     [<Fact>]
     let rootLink_issue275 () =