Skip to content

Commit 6095ae5

Browse files
justinytchenJustin Chen
justinytchen
and
Justin Chen
authored
Added Handler for Semantic Tokenization (#1328)
* added basic semantic token support * removed unnecessary imports * removed unnecessary field * minor refactoring changes * minor refactoring changes * change tokenize to non async * rename handler * refactoring + copyright * renamed handler file * Delete log20200713.txt * moved/refactored handler * added e2e tets * updated test * remove pragma * removed extra spacing * added testing for converting from PS token to semantic tokens * refactored the functions related to converting between tokens * refactored ConvertSemanticToken * fixed tests * added more test cases * fixed spacing * added enum test * fixed spacing issues * fixed spacing, added note about token array representation * changed name to PsesSemanticTokensHandler * reformatted fields * renamed file * used Assert.Collection instead of Assert.Single * modified yml file to fix build * undo changes in yml file * addressed issues in PR * added basic semantic token support * removed unnecessary imports * removed unnecessary field * minor refactoring changes * minor refactoring changes * change tokenize to non async * rename handler * refactoring + copyright * renamed handler file * moved/refactored handler * added e2e tets * Delete log20200713.txt * updated test * remove pragma * removed extra spacing * added testing for converting from PS token to semantic tokens * refactored the functions related to converting between tokens * refactored ConvertSemanticToken * fixed tests * added more test cases * fixed spacing * added enum test * fixed spacing issues * fixed spacing, added note about token array representation * changed name to PsesSemanticTokensHandler * reformatted fields * renamed file * used Assert.Collection instead of Assert.Single * addressed issues in PR * remove unused using * Delete Untitled-1.json Co-authored-by: Justin Chen <[email protected]>
1 parent e326c48 commit 6095ae5

File tree

5 files changed

+415
-0
lines changed

5 files changed

+415
-0
lines changed

Diff for: ‎src/PowerShellEditorServices/Server/PsesLanguageServer.cs

+1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public async Task StartAsync()
9292
.WithHandler<GetCommandHandler>()
9393
.WithHandler<ShowHelpHandler>()
9494
.WithHandler<ExpandAliasHandler>()
95+
.WithHandler<PsesSemanticTokensHandler>()
9596
.OnInitialize(
9697
async (languageServer, request, cancellationToken) =>
9798
{

Diff for: ‎src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
//
2+
// Copyright (c) Microsoft. All rights reserved.
3+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
4+
//
5+
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Management.Automation.Language;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
using Microsoft.Extensions.Logging;
12+
using Microsoft.PowerShell.EditorServices.Services;
13+
using Microsoft.PowerShell.EditorServices.Services.TextDocument;
14+
using Microsoft.PowerShell.EditorServices.Utility;
15+
using OmniSharp.Extensions.LanguageServer.Protocol;
16+
using OmniSharp.Extensions.LanguageServer.Protocol.Document.Proposals;
17+
using OmniSharp.Extensions.LanguageServer.Protocol.Models;
18+
using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
19+
20+
namespace Microsoft.PowerShell.EditorServices.Handlers
21+
{
22+
internal class PsesSemanticTokensHandler : SemanticTokensHandler
23+
{
24+
private static readonly SemanticTokensRegistrationOptions s_registrationOptions = new SemanticTokensRegistrationOptions
25+
{
26+
DocumentSelector = LspUtils.PowerShellDocumentSelector,
27+
Legend = new SemanticTokensLegend(),
28+
DocumentProvider = new Supports<SemanticTokensDocumentProviderOptions>(
29+
isSupported: true,
30+
new SemanticTokensDocumentProviderOptions
31+
{
32+
Edits = true
33+
}),
34+
RangeProvider = true
35+
};
36+
37+
private readonly ILogger _logger;
38+
private readonly WorkspaceService _workspaceService;
39+
40+
public PsesSemanticTokensHandler(ILogger<PsesSemanticTokensHandler> logger, WorkspaceService workspaceService)
41+
: base(s_registrationOptions)
42+
{
43+
_logger = logger;
44+
_workspaceService = workspaceService;
45+
}
46+
47+
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
48+
CancellationToken cancellationToken)
49+
{
50+
ScriptFile file = _workspaceService.GetFile(identifier.TextDocument.Uri);
51+
foreach (Token token in file.ScriptTokens)
52+
{
53+
PushToken(token, builder);
54+
}
55+
return Task.CompletedTask;
56+
}
57+
58+
private static void PushToken(Token token, SemanticTokensBuilder builder)
59+
{
60+
foreach (SemanticToken sToken in ConvertToSemanticTokens(token))
61+
{
62+
builder.Push(
63+
sToken.Line,
64+
sToken.Column,
65+
length: sToken.Text.Length,
66+
sToken.Type,
67+
tokenModifiers: sToken.TokenModifiers);
68+
}
69+
}
70+
71+
internal static IEnumerable<SemanticToken> ConvertToSemanticTokens(Token token)
72+
{
73+
if (token is StringExpandableToken stringExpandableToken)
74+
{
75+
// Try parsing tokens within the string
76+
if (stringExpandableToken.NestedTokens != null)
77+
{
78+
foreach (Token t in stringExpandableToken.NestedTokens)
79+
{
80+
foreach (SemanticToken subToken in ConvertToSemanticTokens(t))
81+
yield return subToken;
82+
}
83+
yield break;
84+
}
85+
}
86+
87+
SemanticTokenType mappedType = MapSemanticTokenType(token);
88+
if (mappedType == null)
89+
{
90+
yield break;
91+
}
92+
93+
//Note that both column and line numbers are 0-based
94+
yield return new SemanticToken(
95+
token.Text,
96+
mappedType,
97+
line: token.Extent.StartLineNumber - 1,
98+
column: token.Extent.StartColumnNumber - 1,
99+
tokenModifiers: Array.Empty<string>());
100+
}
101+
102+
private static SemanticTokenType MapSemanticTokenType(Token token)
103+
{
104+
// First check token flags
105+
if ((token.TokenFlags & TokenFlags.Keyword) != 0)
106+
{
107+
return SemanticTokenType.Keyword;
108+
}
109+
110+
if ((token.TokenFlags & TokenFlags.CommandName) != 0)
111+
{
112+
return SemanticTokenType.Function;
113+
}
114+
115+
if (token.Kind != TokenKind.Generic && (token.TokenFlags &
116+
(TokenFlags.BinaryOperator | TokenFlags.UnaryOperator | TokenFlags.AssignmentOperator)) != 0)
117+
{
118+
return SemanticTokenType.Operator;
119+
}
120+
121+
if ((token.TokenFlags & TokenFlags.TypeName) != 0)
122+
{
123+
return SemanticTokenType.Type;
124+
}
125+
126+
if ((token.TokenFlags & TokenFlags.MemberName) != 0)
127+
{
128+
return SemanticTokenType.Member;
129+
}
130+
131+
// Only check token kind after checking flags
132+
switch (token.Kind)
133+
{
134+
case TokenKind.Comment:
135+
return SemanticTokenType.Comment;
136+
137+
case TokenKind.Parameter:
138+
case TokenKind.Generic when token is StringLiteralToken slt && slt.Text.StartsWith("--"):
139+
return SemanticTokenType.Parameter;
140+
141+
case TokenKind.Variable:
142+
case TokenKind.SplattedVariable:
143+
return SemanticTokenType.Variable;
144+
145+
case TokenKind.StringExpandable:
146+
case TokenKind.StringLiteral:
147+
case TokenKind.HereStringExpandable:
148+
case TokenKind.HereStringLiteral:
149+
return SemanticTokenType.String;
150+
151+
case TokenKind.Number:
152+
return SemanticTokenType.Number;
153+
154+
case TokenKind.Generic:
155+
return SemanticTokenType.Function;
156+
}
157+
158+
return null;
159+
}
160+
161+
protected override Task<SemanticTokensDocument> GetSemanticTokensDocument(
162+
ITextDocumentIdentifierParams @params,
163+
CancellationToken cancellationToken)
164+
{
165+
return Task.FromResult(new SemanticTokensDocument(GetRegistrationOptions().Legend));
166+
}
167+
}
168+
}

Diff for: ‎src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using System.Collections.Generic;
2+
using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
3+
4+
namespace Microsoft.PowerShell.EditorServices.Services.TextDocument
5+
{
6+
internal class SemanticToken
7+
{
8+
public SemanticToken(string text, SemanticTokenType type, int line, int column, IEnumerable<string> tokenModifiers)
9+
{
10+
Line = line;
11+
Text = text;
12+
Column = column;
13+
Type = type;
14+
TokenModifiers = tokenModifiers;
15+
}
16+
17+
public string Text { get; set ;}
18+
19+
public int Line { get; set; }
20+
21+
public int Column { get; set; }
22+
23+
public SemanticTokenType Type { get; set; }
24+
25+
public IEnumerable<string> TokenModifiers { get; set; }
26+
}
27+
}

Diff for: ‎test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs

+31
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using OmniSharp.Extensions.LanguageServer.Protocol.Document;
2020
using OmniSharp.Extensions.LanguageServer.Protocol.Models;
2121
using OmniSharp.Extensions.LanguageServer.Protocol.Workspace;
22+
using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals;
2223
using Xunit;
2324
using Xunit.Abstractions;
2425
using Range = OmniSharp.Extensions.LanguageServer.Protocol.Models.Range;
@@ -1136,5 +1137,35 @@ await PsesLanguageClient
11361137

11371138
Assert.Equal("Get-ChildItem", expandAliasResult.Text);
11381139
}
1140+
1141+
[Fact]
1142+
public async Task CanSendSemanticTokenRequest()
1143+
{
1144+
string scriptContent = "function";
1145+
string scriptPath = NewTestFile(scriptContent);
1146+
1147+
SemanticTokens result =
1148+
await PsesLanguageClient
1149+
.SendRequest<SemanticTokensParams>(
1150+
"textDocument/semanticTokens",
1151+
new SemanticTokensParams
1152+
{
1153+
TextDocument = new TextDocumentIdentifier
1154+
{
1155+
Uri = new Uri(scriptPath)
1156+
}
1157+
})
1158+
.Returning<SemanticTokens>(CancellationToken.None);
1159+
1160+
// More information about how this data is generated can be found at
1161+
// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
1162+
var expectedArr = new int[5]
1163+
{
1164+
// line, index, token length, token type, token modifiers
1165+
0, 0, scriptContent.Length, 2, 0 //function token: line 0, index 0, length, type 2 = keyword, no modifiers
1166+
};
1167+
1168+
Assert.Equal(expectedArr, result.Data.ToArray());
1169+
}
11391170
}
11401171
}

0 commit comments

Comments
 (0)