Syntax highlighting that plays well with font-lock

.azure-pipelines/main.yml

@@ -50,7 +50,10 @@ resources:
 steps:
 - template: .azure-pipelines/steps/setup-rust.yml@emacs-module-rs
 - template: .azure-pipelines/steps/setup-llvm.yml@emacs-module-rs
-- template: steps/-build.yml
 - template: .azure-pipelines/steps/setup-emacs.yml@emacs-module-rs
+- template: .azure-pipelines/steps/setup-cask.yml@emacs-module-rs
+- template: steps/-build.yml
+  parameters:
+    target: release
 - template: steps/setup-tree-sitter-cli.yml
 - template: steps/-test.yml

.azure-pipelines/release.yml

@@ -33,10 +33,11 @@ jobs:
   steps:
   - template: .azure-pipelines/steps/setup-rust.yml@emacs-module-rs
   - template: .azure-pipelines/steps/setup-llvm.yml@emacs-module-rs
+  - template: .azure-pipelines/steps/setup-emacs.yml@emacs-module-rs
+  - template: .azure-pipelines/steps/setup-cask.yml@emacs-module-rs
   - template: steps/-build.yml
     parameters:
       target: release
-  - template: .azure-pipelines/steps/setup-emacs.yml@emacs-module-rs
   - template: steps/setup-tree-sitter-cli.yml
   - template: steps/-test.yml
   - template: steps/-save-binaries.yml

.azure-pipelines/steps/-build.yml

@@ -3,11 +3,13 @@ parameters:
 
 steps:
 - powershell: |
+    cask install
     .\bin\build.ps1 ${{ parameters.target }}
   displayName: Build all packages (Windows)
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
 - bash: |
+    cask install
     ./bin/build ${{ parameters.target }}
   displayName: Build all packages
   condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))

.gitignore

@@ -1,6 +1,6 @@
 /target
 **/*.rs.bk
-**/*.elc
+**/*.elc*
 **/.cask
 grammars/
 lisp/tree-sitter-dyn.*

.travis.yml

@@ -62,9 +62,12 @@ before_install:
   - git clone -b master https://github.com/ubolonton/evm.git $HOME/.evm
   - evm config path /tmp
   - evm install $EVM_EMACS --use --skip
+  - curl -fsSL https://raw.githubusercontent.com/cask/cask/master/go | python
+  - export PATH="$HOME/.cask/bin:$PATH"
 
 install:
-  - make build
+  - cask install
+  - ./bin/build
 
 before_script:
   - nvm install 10

Cargo.toml

@@ -11,7 +11,7 @@ name = "tree_sitter_dyn"
 crate-type = ["cdylib"]
 
 [dependencies]
-emacs = "0.13.0"
+emacs = "0.14.0"
 libloading = "0.5.1"
 tree-sitter = "0.6.3"
 

Cask

@@ -8,3 +8,8 @@
        "Cargo.toml"
        "Cargo.lock"
        "src")
+
+(source melpa)
+
+(development
+ (depends-on "rust-mode"))

README.md

@@ -38,15 +38,15 @@ If you want to hack on `emacs-tree-sitter` itself, see the section [Setup for De
 - Show the debug view of a buffer's parse tree
     ```emacs-lisp
     (require 'tree-sitter-debug)
-    (tree-sitter-debug-enable)
+    (tree-sitter-debug-mode)
     ```
 - Get names of all functions in a Rust file:
     ```emacs-lisp
     (with-current-buffer "types.rs"
       (seq-map (lambda (capture)
                  (pcase-let ((`(_ . ,node) capture))
                    (ts-node-text node)))
-               (tree-sitter-query [(function_item (identifier) @name)])))
+               (tree-sitter-debug-query [(function_item (identifier) @name)])))
     ```
 - Write a simple extension to `expand-region`:
     ```emacs-lisp
@@ -125,12 +125,13 @@ For consistency with Emacs's conventions, this binding has some differences comp
     + `ts-make-query`: create a new query.
     + `ts-make-query-cursor`: create a new query cursor.
     + `ts-query-matches`, `ts-query-captures`: execute a query, returning matches/captures.
-    + `ts-set-byte-range`, `ts-set-point-range`: limit query execution to a range.
 
 ## Setup for Development
 
 Clone this repo and add its `lisp` and `langs` directories to `load-path`.
 
+Install [cask](https://cask.readthedocs.io) and run `cask install` to install dev dependencies.
+
 If you want to hack on the high-level features (in Lisp) only:
 - Evaluate this (once) to download the necessary binaries:
     ```emacs-lisp

bin/build

@@ -20,4 +20,6 @@ source "$here/env.bash"
 
     MODULE_DIR="$PROJECT_ROOT/target/$TARGET"
     cp -f "$MODULE_DIR/$MODULE_ORIGINAL" "./lisp/$MODULE_RENAMED"
+
+    cask build
 )

bin/build.ps1

@@ -17,6 +17,8 @@ Push-Location $project_root
 
 cargo build --all $extra
 
-Pop-Location
-
 Copy-Item $module_dir\$module_name.dll $project_root\lisp\$module_renamed.dll
+
+cask build
+
+Pop-Location

bin/test

@@ -12,7 +12,7 @@ if [[ $* == "watch" ]]; then
         cargo watch -s bin/build -s bin/test
     )
 else
-    $EMACS --batch \
+    cask emacs --batch \
            --directory "$PROJECT_ROOT/lisp" \
            --directory "$PROJECT_ROOT/langs" \
            -l ert \

bin/test.ps1

@@ -19,7 +19,7 @@ if ($args[0] -eq "watch") {
     # https://github.com/PowerShell/PowerShell/issues/4002
     # https://stackoverflow.com/questions/2095088/error-when-calling-3rd-party-executable-from-powershell-when-using-an-ide
     $ErrorActionPreference = 'Continue'
-    emacs --batch `
+    cask emacs --batch `
       --directory "$project_root\lisp" `
       --directory "$project_root\langs" `
       -l ert `

langs/tree-sitter-langs-build.el

@@ -88,7 +88,7 @@ If VERSION and OS are not spcified, use the defaults of
     (php        "v0.16.1")
     (python     "v0.16.0")
     (ruby       "v0.16.1")
-    (rust       "v0.16.0")
+    (rust       "3e5ec5a")
     (scala      "v0.13.0")
     (swift      "a22fa5e")
     (typescript "v0.16.1" ("typescript" "tsx")))
@@ -181,9 +181,10 @@ This function requires git and tree-sitter CLI."
     (if (file-directory-p dir)
         (let ((default-directory dir))
           (tree-sitter-langs--call "git" "remote" "-v" "update"))
-      (tree-sitter-langs--call "git" "clone" "-v" repo dir))
+      (tree-sitter-langs--call "git" "clone" "-q" repo dir))
     (let ((default-directory dir))
       (tree-sitter-langs--call "git" "reset" "--hard" version)
+      (tree-sitter-langs--call "npm" "set" "progress=false")
       ;; TODO: Figure out why we need to skip `npm install' for some repos.
       (ignore-errors
         (tree-sitter-langs--call "npm" "install"))

langs/tree-sitter-langs.el

@@ -11,20 +11,37 @@
 
 ;;; Commentary:
 
-;; This is a convenient bundle of language grammars for `tree-sitter'. It serves
-;; as an interim distribution mechanism, until `tree-sitter' is widespread
-;; enough for language major modes to include the definitions on their own.
+;; This is a convenient bundle of language grammars and queries for
+;; `tree-sitter'. It serves as an interim distribution mechanism, until
+;; `tree-sitter' is widespread enough for language major modes to include these
+;; definitions on their own.
+;;
+;; Basically it's a multi-step process:
+;;
+;; 1. `tree-sitter-langs' populates global registries of grammars and queries.
+;;    These global registries are defined by `tree-sitter-mode' and other
+;;    `tree-sitter'-based language-agnostic minor modes, to extend existing
+;;    major modes.
+;;
+;; 2. New `tree-sitter'-based language-specific minor modes use these global
+;;    registries to extend existing major modes.
+;;
+;; 3. Major modes adopt new `tree-sitter'-based features, and distribute the
+;;    grammars and queries on their own. They can either put these definitions
+;;    in the global registries, or keep using them only internally.
 
 ;;; Code:
 
+(require 'cl-lib)
+
 (require 'tree-sitter)
 (require 'tree-sitter-load)
+(require 'tree-sitter-hl)
 
 (require 'tree-sitter-langs-build)
 
 (eval-when-compile
-  (require 'pcase)
-  (require 'cl-lib))
+  (require 'pcase))
 
 (defun tree-sitter-langs-ensure (lang-symbol)
   "Return the language object identified by LANG-SYMBOL.
@@ -71,5 +88,26 @@ See `tree-sitter-langs-repos'."
   (setf (map-elt tree-sitter-major-mode-language-alist major-mode)
         lang-symbol))
 
+(defun tree-sitter-langs--hl-default-patterns (lang-symbol)
+  "Return default syntax highlighting patterns for LANG-SYMBOL."
+  (let ((query-path (cl-reduce
+                     (lambda (dir name) (expand-file-name name dir))
+                     `("repos" ,(format "tree-sitter-%s" lang-symbol)
+                       "queries" "highlights.scm")
+                     :initial-value tree-sitter-langs--dir)))
+    (with-temp-buffer
+      (insert-file-contents query-path)
+      (buffer-string))))
+
+(defun tree-sitter-langs--set-hl-default-patterns (&rest _args)
+  "Use syntax highlighting patterns provided by `tree-sitter-langs'."
+  (unless tree-sitter-hl-default-patterns
+    (let ((lang-symbol (alist-get major-mode tree-sitter-major-mode-language-alist)))
+      (setq tree-sitter-hl-default-patterns
+            (tree-sitter-langs--hl-default-patterns lang-symbol)))))
+
+(advice-add 'tree-sitter-hl--setup :before
+            #'tree-sitter-langs--set-hl-default-patterns)
+
 (provide 'tree-sitter-langs)
 ;;; tree-sitter-langs.el ends here

lisp/test-files/change-case-region.rs

@@ -0,0 +1,5 @@
+// Change case of "this text" repeatedly.
+unsafe {
+    input_function.call_unprotected((bytepos, point.line_number(), point.byte_column()))
+        .and_then(|v| v.into_rust())
+}

lisp/test-files/delete-non-ascii-text.rs

@@ -0,0 +1,5 @@
+// Delete the 8 chars next line.
+// ấấấấấấấấ
+unsafe {
+    input_function.call_unprotected((bytepos, point.line_number(), point.byte_column()))
+}

lisp/test-files/extend-region.rs

@@ -0,0 +1,6 @@
+macro_rules! impl_pred {}
+
+// In evil's normal-mode, with point after `i`, or `;`, or at the end of the file, eval
+// (font-lock-flush). A correct implementation would highlight `i` and `!`.
+abc
+impl_pred!(foo, bar);