Merge pull request #245 from tayloraswift/indexstore-highlighting

improved indexstore-powered highlights

Author: tayloraswift

.github/pipeline

@@ -1,7 +1,13 @@
 #!/bin/bash
 set -e
+
 swift --version
-swift build -c release --explicit-target-dependency-import-check=error
+
+swift build -c release \
+    --explicit-target-dependency-import-check=error \
+    -Xcxx -I/usr/lib/swift \
+    -Xcxx -I/usr/lib/swift/Block
+
 ./generate-test-symbolgraphs
 for f in .build/release/*Tests; do
   $f

.github/workflows/ci.yml

@@ -54,12 +54,17 @@ jobs:
             -   name: Build debug
                 run: |
                     swift --version
-                    swift build
+                    swift build -Xcxx -I/usr/lib/swift -Xcxx -I/usr/lib/swift/Block
 
             -   name: Build release
                 run: |
-                    swift build -c release
+                    swift build -c release \
+                        -Xcxx -I/usr/lib/swift \
+                        -Xcxx -I/usr/lib/swift/Block
 
             -   name: Test SymbolGraphBuilder
                 run: |
-                    swift run -c release SymbolGraphBuilderTests
+                    swift run -c release \
+                        -Xcxx -I/usr/lib/swift \
+                        -Xcxx -I/usr/lib/swift/Block \
+                        SymbolGraphBuilderTests

Package.resolved

@@ -1,5 +1,5 @@
 {
-  "originHash" : "25883f12cf4eb759ed4a7b02bac9272cad664cd8aabefb3cc5928b2793c7845f",
+  "originHash" : "11a2c92dc12e2d1090f389f23c98adcb98ad3e0b6ccf1e724d65aeb0152c5554",
   "pins" : [
     {
       "identity" : "swift-atomics",

Package.swift

@@ -1,9 +1,10 @@
 // swift-tools-version:5.10
+import class Foundation.ProcessInfo
 import PackageDescription
 import CompilerPluginSupport
 
 let package:Package = .init(
-    name: "swift-unidoc",
+    name: "Swift Unidoc",
     platforms: [.macOS(.v14)],
     products: [
         .executable(name: "ssgc", targets: ["ssgc"]),
@@ -102,9 +103,6 @@ let package:Package = .init(
         .package(url: "https://github.com/tayloraswift/swift-png", .upToNextMinor(
             from: "4.4.2")),
 
-        // .package(url: "https://github.com/apple/indexstore-db",
-        //     branch: "swift-5.10-RELEASE"),
-
         .package(url: "https://github.com/apple/swift-atomics", .upToNextMinor(
             from: "1.2.0")),
         .package(url: "https://github.com/apple/swift-collections.git", .upToNextMinor(
@@ -296,18 +294,13 @@ let package:Package = .init(
                 .target(name: "MarkdownABI"),
                 .target(name: "Signatures"),
                 .target(name: "Snippets"),
+                .target(name: "Sources"),
                 .target(name: "Symbols"),
 
                 .product(name: "SwiftIDEUtils", package: "swift-syntax"),
                 .product(name: "SwiftParser", package: "swift-syntax"),
             ]),
 
-        .target(name: "MarkdownPluginSwift_IndexStoreDB",
-            dependencies: [
-                .target(name: "MarkdownPluginSwift"),
-                // .product(name: "IndexStoreDB", package: "indexstore-db"),
-            ]),
-
         .target(name: "MarkdownSemantics",
             dependencies: [
                 .target(name: "MarkdownAST"),
@@ -756,6 +749,25 @@ let package:Package = .init(
         .target(name: "guides", path: "Guides"),
     ])
 
+switch ProcessInfo.processInfo.environment["UNIDOC_ENABLE_INDEXSTORE"]?.lowercased()
+{
+case "1"?, "true"?:
+    package.dependencies.append(.package(url: "https://github.com/apple/indexstore-db",
+        branch: "swift-5.10-RELEASE"))
+
+    package.targets.append(.target(name: "MarkdownPluginSwift_IndexStoreDB",
+        dependencies: [
+            .target(name: "MarkdownPluginSwift"),
+            .product(name: "IndexStoreDB", package: "indexstore-db"),
+        ]))
+
+default:
+    package.targets.append(.target(name: "MarkdownPluginSwift_IndexStoreDB",
+        dependencies: [
+            .target(name: "MarkdownPluginSwift"),
+        ]))
+}
+
 for target:PackageDescription.Target in package.targets
 {
     if  target.name == "_AsyncChannel"

Sources/MarkdownPluginSwift/Markdown.SwiftLanguage.IndexMarker.swift

@@ -0,0 +1,32 @@
+import Symbols
+import Sources
+
+extension Markdown.SwiftLanguage
+{
+    @frozen public
+    struct IndexMarker
+    {
+        public
+        let position:SourcePosition
+        public
+        let symbol:Symbol.USR
+        public
+        let phylum:Phylum.Decl?
+
+        @inlinable public
+        init(position:SourcePosition, symbol:Symbol.USR, phylum:Phylum.Decl?)
+        {
+            self.position = position
+            self.symbol = symbol
+            self.phylum = phylum
+        }
+    }
+}
+extension Markdown.SwiftLanguage.IndexMarker:CustomStringConvertible
+{
+    public
+    var description:String
+    {
+        "(\(self.position): \(self.symbol), \(self.phylum?.name ?? "unknown"))"
+    }
+}

Sources/MarkdownPluginSwift/Markdown.SwiftLanguage.IndexStore.swift

@@ -3,6 +3,7 @@ extension Markdown.SwiftLanguage
     public
     protocol IndexStore:AnyObject
     {
-        func load(for path:String)
+        /// Returns a list of index markers, indexed by UTF-8 offset.
+        func load(for path:String, utf8:[UInt8]) -> [Int: IndexMarker]
     }
 }

Sources/MarkdownPluginSwift/Markdown.SwiftLanguage.swift

@@ -34,9 +34,16 @@ extension Markdown.SwiftLanguage
     func parse(snippet utf8:[UInt8],
         from indexID:String? = nil) -> (caption:String, slices:[Markdown.SnippetSlice])
     {
-        if  let indexID:String
+        let links:[Int: IndexMarker]
+
+        if  let indexID:String,
+            let index:any IndexStore = self.index
+        {
+            links = index.load(for: indexID, utf8: utf8)
+        }
+        else
         {
-            self.index?.load(for: indexID)
+            links = [:]
         }
 
         //  It is safe to escape the pointer to ``Parser.parse(source:maximumNestingLevel:)``,
@@ -98,7 +105,7 @@ extension Markdown.SwiftLanguage
         }
 
         let slices:[SnippetParser.Slice] = parser.finish(at: parsed.endPosition, in: utf8)
-        var cursor:SyntaxClassificationCursor = .init(parsed.classifications)
+        var cursor:SyntaxClassificationCursor = .init(parsed.classifications, links: links)
 
         let rendered:[Markdown.SnippetSlice] = slices.map
         {

Sources/MarkdownPluginSwift/SyntaxClassificationCursor.SpanIterator.swift

@@ -0,0 +1,69 @@
+import SwiftIDEUtils
+import Symbols
+
+extension SyntaxClassificationCursor
+{
+    struct SpanIterator
+    {
+        private
+        var links:[Int: Markdown.SwiftLanguage.IndexMarker]
+        private
+        var spans:Array<SyntaxClassifiedRange>.Iterator
+
+        init(_ spans:consuming SyntaxClassifications,
+            links:[Int: Markdown.SwiftLanguage.IndexMarker] = [:])
+        {
+            self.links = links
+            self.spans = spans.makeIterator()
+        }
+    }
+}
+extension SyntaxClassificationCursor.SpanIterator:CustomDebugStringConvertible
+{
+    var debugDescription:String
+    {
+        self.links
+            .sorted
+        {
+            $0.key < $1.key
+        }
+            .map
+        {
+            "[\($0.key)]: \($0.value)"
+        }
+            .joined(separator: "\n")
+    }
+}
+extension SyntaxClassificationCursor.SpanIterator
+{
+    mutating
+    func next() -> SyntaxClassifiedRange?
+    {
+        guard
+        var highlight:SyntaxClassifiedRange = self.spans.next()
+        else
+        {
+            return nil
+        }
+
+        if  let marker:Markdown.SwiftLanguage.IndexMarker = self.links[highlight.offset],
+            let phylum:Phylum.Decl = marker.phylum
+        {
+            switch phylum
+            {
+            case .actor:            highlight.kind = .type
+            case .associatedtype:   highlight.kind = .type
+            case .class:            highlight.kind = .type
+            case .enum:             highlight.kind = .type
+            case .protocol:         highlight.kind = .type
+            case .struct:           highlight.kind = .type
+            case .typealias:        highlight.kind = .type
+            case .func:             highlight.kind = .identifier
+            case .initializer:      highlight.kind = .keyword
+            default:                break
+            }
+        }
+
+        return highlight
+    }
+}

Sources/MarkdownPluginSwift/SyntaxClassificationCursor.swift

@@ -3,12 +3,15 @@ import SwiftSyntax
 
 struct SyntaxClassificationCursor
 {
-    var spans:SyntaxClassifications.Iterator
+    private(set)
+    var spans:SpanIterator
+    private
     var span:SyntaxClassifiedRange?
 
-    init(_ spans:consuming SyntaxClassifications)
+    init(_ spans:consuming SyntaxClassifications,
+        links:[Int: Markdown.SwiftLanguage.IndexMarker] = [:])
     {
-        self.spans = spans.makeIterator()
+        self.spans = .init(spans, links: links)
         self.span = self.spans.next()
     }
 

Sources/MarkdownPluginSwift_IndexStoreDB/IndexStoreDB (ext).swift

@@ -1,20 +1,89 @@
 #if canImport(IndexStoreDB)
 
-import IndexStoreDB
+import class IndexStoreDB.IndexStoreDB
 import MarkdownPluginSwift
+import Sources
+import Symbols
 
 extension IndexStoreDB:Markdown.SwiftLanguage.IndexStore
 {
     public
-    func load(for id:String)
+    func load(for id:String, utf8:[UInt8]) -> [Int: Markdown.SwiftLanguage.IndexMarker]
     {
-        for symbol:Symbol in self.symbols(inFilePath: id)
+        //  Compute line positions
+        var lines:[Int: Int] = [1: utf8.startIndex]
+        var line:Int = 1
+        for (i, byte):(Int, UInt8) in zip(utf8.indices, utf8)
         {
-            for occurence:SymbolOccurrence in self.occurrences(ofUSR: symbol.usr, roles: .all)
+            if  byte == 0x0A
             {
-                print(occurence)
+                line += 1
+                lines[line] = utf8.index(after: i)
             }
         }
+
+        var markers:[Int: Markdown.SwiftLanguage.IndexMarker] = [:]
+        for symbol:IndexStoreDB.Symbol_ in self.symbols(inFilePath: id)
+        {
+            for occurence:IndexStoreDB.SymbolOccurrence_ in self.occurrences(ofUSR: symbol.usr,
+                roles: .all)
+            {
+                guard
+                let position:SourcePosition = .init(line: occurence.location.line - 1,
+                    column: occurence.location.utf8Column - 1),
+                let base:Int = lines[occurence.location.line],
+                let usr:Symbol.USR = .init(symbol.usr)
+                else
+                {
+                    continue
+                }
+
+                let phylum:Phylum.Decl?
+
+                switch symbol.kind
+                {
+                case .constructor:
+                    phylum = occurence.roles.contains(.call)
+                        ? .func(.static)
+                        : .initializer
+
+                case .unknown:              phylum = nil
+                case .module:               phylum = nil
+                case .namespace:            phylum = nil
+                case .namespaceAlias:       phylum = nil
+                case .macro:                phylum = nil
+                case .enum:                 phylum = .enum
+                case .struct:               phylum = .struct
+                case .class:                phylum = .class
+                case .protocol:             phylum = .protocol
+                case .extension:            phylum = .typealias
+                case .union:                phylum = .enum
+                case .typealias:            phylum = .typealias
+                case .function:             phylum = .func(nil)
+                case .variable:             phylum = .var(nil)
+                case .field:                phylum = nil
+                case .enumConstant:         phylum = .case
+                case .instanceMethod:       phylum = .func(.instance)
+                case .classMethod:          phylum = .func(.class)
+                case .staticMethod:         phylum = .func(.static)
+                case .instanceProperty:     phylum = .var(.instance)
+                case .classProperty:        phylum = .var(.class)
+                case .staticProperty:       phylum = .var(.static)
+                case .destructor:           phylum = .deinitializer
+                case .conversionFunction:   phylum = nil
+                case .parameter:            phylum = nil
+                case .using:                phylum = nil
+                case .concept:              phylum = nil
+                case .commentTag:           phylum = nil
+                }
+
+                markers[base + occurence.location.utf8Column - 1] = .init(position: position,
+                    symbol: usr,
+                    phylum: phylum)
+            }
+        }
+
+        return markers
     }
 }
 

Sources/MarkdownPluginSwift_IndexStoreDB/shims.swift

@@ -0,0 +1,11 @@
+#if canImport(IndexStoreDB)
+
+import IndexStoreDB
+
+extension IndexStoreDB
+{
+    typealias Symbol_ = Symbol
+    typealias SymbolOccurrence_ = SymbolOccurrence
+}
+
+#endif