diff --git a/README.md b/README.md index 65ace8408a0..0b2a606243b 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,7 @@ most popular Emacs packages like `company`, `flycheck` and `projectile`. - Ivy integration - [lsp-ivy](https://github.com/emacs-lsp/lsp-ivy/) - Treemacs integration - [lsp-treemacs](https://github.com/emacs-lsp/lsp-treemacs) - - Semantic highlighting (as currently implemented by JDT LS and - unreleased builds of clangd, cf. [Semantic highlighting - spec](https://github.com/microsoft/vscode-languageserver-node/pull/367)) + - Semantic highlighting (using either the theia semantic highlighting protocol as implemented by JDT LS and some versions of clangd, or the semantic tokens protocol defined by LSP 3.16) - [which-key](https://github.com/justbur/emacs-which-key/) integration for better discovery diff --git a/lsp-mode.el b/lsp-mode.el index 21ec084ac59..7a2ff1c5b83 100644 --- a/lsp-mode.el +++ b/lsp-mode.el @@ -170,28 +170,37 @@ the buffer when it becomes large." :package-version '(lsp-mode . "6.1")) (defcustom lsp-semantic-highlighting nil - "When set to `:immediate' or `:deferred', this option enables - semantic highlighting as proposed at - https://github.com/microsoft/vscode-languageserver-node/pull/367. - - If `lsp-semantic-highlighting' is set to `:immediate', semantic - highlighting information received from the language server is - applied immediately, and in full. If `lsp-semantic-highlighting' - is set to `:deferred', semantic highlighting will be performed - after an idle timeout, and only within a limit region - around `(point)' (see - `lsp-semantic-highlighting-context-lines'). Compared to - `:immediate', `:deferred' has a higher risk of producing stale - highlights but may offer significantly better performance. - - Note that semantic highlighting is not yet part of the official - LSP spec and may occasionally break as language servers are - updated." + "When set to `:semantic-tokens', this option enables support +for semantic highlighting as defined by the Language Server +Protocol 3.16. + +Some older language servers may not conform to the semantic +tokens protocol yet, but implement the theia semantic +highlighting protocol, see +https://github.com/microsoft/vscode-languageserver-node/pull/367. +To use theia highlighting, set `lsp-semantic-highlighting' to +`:immediate' or `:deferred', where `:deferred' should offer +better performance than `:immediate' but has a higher risk of +producing stale highlights." :group 'lsp-mode :type '(choice (const :tag "Disable" nil) (const :tag "Immediate" :immediate) - (const :tag "Deferred" :deferred))) + (const :tag "Deferred" :deferred) + (const :tag "SemanticTokens" :semantic-tokens))) + +(defcustom lsp-semantic-highlighting-warn-on-missing-face nil + "When non-nil, this option will emit a warning any time a token +or modifier type returned by a language server has no face associated with it." + :group 'lsp-mode + :type 'boolean) + +(defcustom lsp-semantic-tokens-apply-modifiers nil + "Determines whether semantic highlighting should take token +modifiers into account. Only applies if +`lsp-semantic-highlighting' is set to `:semantic-tokens'." + :group 'lsp-mode + :type 'boolean) (defcustom lsp-semantic-highlighting-context-lines 15 "How many lines to fontify above `(window-start)' and below `(window-end)'." @@ -740,6 +749,7 @@ directory") ("textDocument/rangeFormatting" :capability "documentRangeFormattingProvider") ("textDocument/references" :capability "referencesProvider") ("textDocument/selectionRange" :capability "selectionRangeProvider") + ("textDocument/semanticTokens" :capability "semanticTokensProvider") ("textDocument/signatureHelp" :capability "signatureHelpProvider") ("textDocument/typeDefinition" :capability "typeDefinitionProvider") ("workspace/executeCommand" :capability "executeCommandProvider") @@ -2609,12 +2619,17 @@ active `major-mode', or for all major modes when ALL-MODES is t." ;; ‘buffers’ is a list of buffers associated with this workspace. (buffers nil) - ;; ‘semantic-highlighting-faces' is a vector containing one face for each - ;; TextMate scope (or set of scopes) supported by the language server. Cf. - ;; ‘lsp-semantic-highlighting-faces' if you wish to change the default - ;; semantic highlighting faces + ;; If old-style (Theia) semantic highlighting is used (i.e., if 'lsp-semantic-highlighting‘ + ;; is set to :immediate or :deferred), semantic-highlighting-faces' is a vector containing + ;; one face for each TextMate scope (or set of scopes) supported by the language server. If + ;; semanticTokens are used for highlighting, semantic-highlighting-faces contains one face + ;; (or nil) for each token type supported by the language server. (semantic-highlighting-faces nil) + ;; If semanticTokens are used for highlighting, semantic-highlighting-modifier-faces contains + ;; one face (or nil) for each modifier type supported by the language server + (semantic-highlighting-modifier-faces nil) + ;; Extra client capabilities provided by third-party packages using ;; `lsp-register-client-capabilities'. It's value is an alist of (PACKAGE-NAME ;; . CAPS), where PACKAGE-NAME is a symbol of the third-party package name, @@ -3132,8 +3147,14 @@ disappearing, unset all the variables related to it." (hierarchicalDocumentSymbolSupport . t))) (formatting . ((dynamicRegistration . t))) (rangeFormatting . ((dynamicRegistration . t))) + ,@(pcase lsp-semantic-highlighting + ((or :immediate :deferred) '((semanticHighlightingCapabilities . ((semanticHighlighting . t))))) + (:semantic-tokens `((semanticTokens + . ((tokenModifiers . ,(if lsp-semantic-tokens-apply-modifiers + (apply 'vector (mapcar #'car lsp-semantic-token-modifier-faces)) [])) + (tokenTypes . ,(apply 'vector (mapcar #'car lsp-semantic-token-faces))))))) + (_ '())) (rename . ((dynamicRegistration . t) (prepareSupport . t))) - (semanticHighlightingCapabilities . ((semanticHighlighting . ,(lsp-json-bool lsp-semantic-highlighting)))) (codeAction . ((dynamicRegistration . t) (isPreferredSupport . t) (codeActionLiteralSupport . ((codeActionKind . ((valueSet . ["" @@ -3453,6 +3474,9 @@ in that particular folder." (lsp--update-on-type-formatting-hook) (lsp--update-signature-help-hook) + (when (and (eq lsp-semantic-highlighting :semantic-tokens) + (lsp-feature? "textDocument/semanticTokens")) + (lsp--semantic-tokens-initialize-buffer)) (add-hook 'post-command-hook #'lsp--post-command nil t) (when lsp-enable-xref (add-hook 'xref-backend-functions #'lsp--xref-backend nil t)) @@ -3494,6 +3518,10 @@ in that particular folder." (remove-hook 'lsp-on-idle-hook #'lsp--document-links t) (remove-hook 'lsp-on-idle-hook #'lsp--document-highlight t) + (when lsp--semantic-tokens-teardown + (funcall lsp--semantic-tokens-teardown) + (setq lsp--semantic-tokens-teardown nil)) + (lsp--remove-overlays 'lsp-sem-highlight) (lsp--remove-overlays 'lsp-highlight) (lsp--remove-overlays 'lsp-links) @@ -5541,8 +5569,145 @@ unless overridden by a more specific face association." Since the list is traversed in order, it should be sorted in order of decreasing specificity.") +(defface lsp-face-semhl-comment + '((t (:inherit font-lock-comment-face))) + "Face used for comments." + :group 'lsp-faces) + +(defface lsp-face-semhl-keyword + '((t (:inherit font-lock-keyword-face))) + "Face used for keywords." + :group 'lsp-faces) + +(defface lsp-face-semhl-string + '((t (:inherit font-lock-string-face))) + "Face used for keywords." + :group 'lsp-faces) + +(defface lsp-face-semhl-number + '((t (:inherit font-lock-constant-face))) + "Face used for numbers." + :group 'lsp-faces) + +(defface lsp-face-semhl-regexp + '((t (:inherit font-lock-string-face :slant italic))) + "Face used for regexps." + :group 'lsp-faces) + +(defface lsp-face-semhl-operator + '((t (:inherit font-lock-function-name-face))) + "Face used for operators." + :group 'lsp-faces) + +(defface lsp-face-semhl-namespace + '((t (:inherit font-lock-keyword-face))) + "Face used for namespaces." + :group 'lsp-faces) + +(defface lsp-face-semhl-type + '((t (:inherit font-lock-type-face))) + "Face used for types." + :group 'lsp-faces) + +(defface lsp-face-semhl-type + '((t (:inherit font-lock-type-face))) + "Face used for types." + :group 'lsp-faces) + +(defface lsp-face-semhl-struct + '((t (:inherit font-lock-type-face))) + "Face used for structs." + :group 'lsp-faces) + +(defface lsp-face-semhl-class + '((t (:inherit font-lock-type-face))) + "Face used for classes." + :group 'lsp-faces) + +(defface lsp-face-semhl-interface + '((t (:inherit font-lock-type-face))) + "Face used for interfaces." + :group 'lsp-faces) + +(defface lsp-face-semhl-enum + '((t (:inherit font-lock-variable-name-face))) + "Face used for enums." + :group 'lsp-faces) + +(defface lsp-face-semhl-type-parameter + '((t (:inherit font-lock-type-face))) + "Face used for type parameters." + :group 'lsp-faces) + +;; function face already defined, move here when support +;; for theia highlighting gets removed +(defface lsp-face-semhl-member + '((t (:inherit font-lock-variable-name-face))) + "Face used for members." + :group 'lsp-faces) + +(defface lsp-face-semhl-property + '((t (:inherit font-lock-variable-name-face))) + "Face used for properties." + :group 'lsp-faces) + +(defface lsp-face-semhl-macro + '((t (:inherit font-lock-preprocessor-face))) + "Face used for macros." + :group 'lsp-faces) + +(defface lsp-face-semhl-variable + '((t (:inherit font-lock-variable-name-face))) + "Face used for variables." + :group 'lsp-faces) + +(defface lsp-face-semhl-parameter + '((t (:inherit font-lock-variable-name-face))) + "Face used for parameters." + :group 'lsp-faces) + +(defface lsp-face-semhl-label + '((t (:inherit font-lock-comment-face))) + "Face used for labels." + :group 'lsp-faces) + +(defvar lsp-semantic-token-faces + '(("comment" . lsp-face-semhl-comment) + ("keyword" . lsp-face-semhl-keyword) + ("string" . lsp-face-semhl-string) + ("number" . lsp-face-semhl-number) + ("regexp" . lsp-face-semhl-regexp) + ("operator" . lsp-face-semhl-operator) + ("namespace" . lsp-face-semhl-namespace) + ("type" . lsp-face-semhl-type) + ("struct" . lsp-face-semhl-struct) + ("class" . lsp-face-semhl-class) + ("interface" . lsp-face-semhl-interface) + ("enum" . lsp-face-semhl-enum) + ("typeParameter" . lsp-face-semhl-type-parameter) + ("function" . lsp-face-semhl-function) + ("member" . lsp-face-semhl-member) + ("property" . lsp-face-semhl-property) + ("macro" . lsp-face-semhl-macro) + ("variable" . lsp-face-semhl-variable) + ("parameter" . lsp-face-semhl-parameter) + ("label" . lsp-face-semhl-label) + ("enumConstant" . lsp-face-semhl-constant) + ("dependent" . lsp-face-semhl-type) + ("concept" . lsp-face-semhl-interface)) + "Faces to use for semantic highlighting if +`lsp-semantic-highlighting' is set to :semantic-tokens.") + +(defvar lsp-semantic-token-modifier-faces + ;; TODO: add default definitions + '(("declaration" . lsp-face-semhl-interface) + ("readonly" . lsp-face-semhl-constant)) + "Faces to use for semantic token modifiers if +`lsp-semantic-highlighting' is set to `:semantic-tokens' and +`lsp-semantic-tokens-apply-modifiers' is non-nil.") + (defvar-local lsp--semantic-highlighting-current-region nil -"Denotes the region `(min . max)' most recently fontified via the + "Denotes the region `(min . max)' most recently fontified via the deferred semantic-highlighting mechanism. Further fontification calls will be skipped unless new semantic @@ -5694,7 +5859,7 @@ or `(point)' lies outside `lsp--semantic-highlighting-region'.") (inhibit-field-text-motion t)) (when buffer (with-current-buffer buffer - (if (eq lsp-semantic-highlighting 'immediate) + (if (eq lsp-semantic-highlighting :immediate) (save-mark-and-excursion (save-restriction (widen) @@ -5703,6 +5868,129 @@ or `(point)' lies outside `lsp--semantic-highlighting-region'.") (lsp--workspace-semantic-highlighting-faces workspace) lines)))) (lsp--semantic-highlighting-add-to-cache lines)))))) +(defun lsp--build-face-map (identifiers faces category varname) + (apply 'vector + (mapcar (lambda (id) + (let ((maybe-face (cdr (assoc id faces)))) + (when (and lsp-semantic-highlighting-warn-on-missing-face (not maybe-face)) + (lsp-warn "No face has been associated to the %s '%s': consider adding a corresponding definition to %s" + category id varname)) maybe-face)) identifiers))) + +(defun lsp--semantic-tokens-initialize-workspace (workspace) + (cl-assert workspace) + (let* ((token-capabilities (gethash + "semanticTokensProvider" + (lsp--workspace-server-capabilities workspace))) + (legend (gethash "legend" token-capabilities))) + (setf (lsp--workspace-semantic-highlighting-faces workspace) + (lsp--build-face-map (gethash "tokenTypes" legend) + lsp-semantic-token-faces + "semantic token" + "lsp-semantic-token-faces")) + (setf (lsp--workspace-semantic-highlighting-modifier-faces workspace) + (lsp--build-face-map (gethash "tokenModifiers" legend) + lsp-semantic-token-modifier-faces + "semantic token modifier" + "lsp-semantic-token-modifier-faces")))) + +(defvar-local lsp--semantic-tokens-cache nil) + +(defvar-local lsp--semantic-tokens-teardown nil) + +(defun lsp--semantic-tokens-initialize-buffer () + (let* ((old-extend-region-functions font-lock-extend-region-functions) + ;; make sure font-lock always fontifies entire lines (TODO: do we also have + ;; to change some jit-lock-...-region functions/variables?) + (new-extend-region-functions + (if (memq 'font-lock-extend-region-wholelines old-extend-region-functions) + old-extend-region-functions + (cons 'font-lock-extend-region-wholelines old-extend-region-functions)))) + (setq font-lock-extend-region-functions new-extend-region-functions) + (add-function :around (local 'font-lock-fontify-region-function) #'lsp--semantic-tokens-fontify) + (add-hook 'lsp-on-change-hook #'lsp--semantic-tokens-request nil t) + (lsp--semantic-tokens-request) + (setq lsp--semantic-tokens-teardown + (lambda () + (setq font-lock-extend-region-functions old-extend-region-functions) + (remove-function (local 'font-lock-fontify-region-function) + #'lsp--semantic-tokens-fontify))))) + +(defun lsp--semantic-tokens-fontify (old-fontify-region beg end &optional loudly) + ;; TODO: support multiple language servers per buffer? + (let ((faces (seq-some #'lsp--workspace-semantic-highlighting-faces lsp--buffer-workspaces)) + (modifier-faces + (when lsp-semantic-tokens-apply-modifiers + (seq-some #'lsp--workspace-semantic-highlighting-modifier-faces lsp--buffer-workspaces)))) + (if (or (eq nil lsp--semantic-tokens-cache) + (eq nil faces) + ;; delay fontification until we have fresh tokens + (not (= lsp--cur-version (gethash "documentVersion" lsp--semantic-tokens-cache)))) + '(jit-lock-bounds 0 . 0) + (funcall old-fontify-region beg end loudly) + (let* ((inhibit-field-text-motion t) + (data (gethash "data" lsp--semantic-tokens-cache)) + (i0 0) + (i-max (1- (length data))) + (current-line 1) + (line-delta) + (column 0) + (face) + (line-start-pos) + (line-min) + (line-max-inclusive) + (text-property-beg) + (text-property-end)) + (save-mark-and-excursion + (save-restriction + (widen) + (goto-char beg) + (goto-char (line-beginning-position)) + (setq line-min (line-number-at-pos)) + (with-silent-modifications + (goto-char end) + (goto-char (line-end-position)) + (setq line-max-inclusive (line-number-at-pos)) + (forward-line (- line-min line-max-inclusive)) + (let ((skip-lines (- line-min current-line))) + (while (and (<= i0 i-max) (< (aref data i0) skip-lines)) + (setq skip-lines (- skip-lines (aref data i0))) + (setq i0 (+ i0 5))) + (setq current-line (- line-min skip-lines))) + (forward-line (- current-line line-min)) + (setq line-start-pos (point)) + (cl-loop + for i from i0 to i-max by 5 do + (setq line-delta (aref data i)) + (unless (= line-delta 0) + (forward-line line-delta) + (setq line-start-pos (point)) + (setq column 0) + (setq current-line (+ current-line line-delta))) + (setq column (+ column (aref data (1+ i)))) + (setq face (aref faces (aref data (+ i 3)))) + (setq text-property-beg (+ line-start-pos column)) + (setq text-property-end (+ text-property-beg (aref data (+ i 2)))) + (when face (put-text-property text-property-beg text-property-end 'face face)) + (cl-loop for i from 0 to (1- (length modifier-faces)) do + (when (and (aref modifier-faces i) + (> 0 (logand (aref data (+ i 4)) (lsh 1 i)))) + (add-face-text-property text-property-beg text-property-end + (aref modifier-faces i)))) + when (> current-line line-max-inclusive) return nil))))) + `(jit-lock-bounds ,beg . ,end)))) + +(defun lsp--semantic-tokens-request () + (let ((cur-version lsp--cur-version)) + (lsp-request-async + "textDocument/semanticTokens" + `(:textDocument ,(lsp--text-document-identifier)) + (lambda (response) + (setq lsp--semantic-tokens-cache response) + (puthash "documentVersion" cur-version lsp--semantic-tokens-cache) + (font-lock-flush)) + :mode 'tick + :cancel-token (format "semantic-tokens-%s" (lsp--buffer-uri))))) + (defconst lsp--symbol-kind '((1 . "File") (2 . "Module") @@ -6919,6 +7207,9 @@ SESSION is the active session." (setf (lsp--workspace-server-capabilities workspace) (gethash "capabilities" response) (lsp--workspace-status workspace) 'initialized) + (mapc #'lsp--semantic-tokens-initialize-workspace + (lsp--find-workspaces-for "textDocument/semanticTokens")) + (with-lsp-workspace workspace (lsp-notify "initialized" lsp--empty-ht))