Highlighting: add tree-sitter support for the most common programming languages

cmd/frontend/internal/highlight/language.go

@@ -21,6 +21,17 @@ const (
 	EngineSyntect
 )
 
+func (e EngineType) String() string {
+	switch e {
+	case EngineTreeSitter:
+		return "tree-sitter"
+	case EngineSyntect:
+		return "syntect"
+	default:
+		return "invalid"
+	}
+}
+
 // Converts an engine type to the corresponding parameter value for the syntax
 // highlighting request. Defaults to "syntec".
 func getEngineParameter(engine EngineType) string {
@@ -59,6 +70,8 @@ var highlightConfig = syntaxHighlightConfig{
 }
 var baseHighlightConfig = syntaxHighlightConfig{
 	Extensions: map[string]string{
+		"jsx":  "jsx", // default `getLanguage()` helper doesn't handle JSX
+		"tsx":  "tsx", // default `getLanguage()` helper doesn't handle TSX
 		"sbt":  "scala",
 		"sc":   "scala",
 		"xlsg": "xlsg",
@@ -91,10 +104,21 @@ var engineConfig = syntaxEngineConfig{
 var baseEngineConfig = syntaxEngineConfig{
 	Default: EngineSyntect,
 	Overrides: map[string]EngineType{
-		"scala":   EngineTreeSitter,
-		"c#":      EngineTreeSitter,
-		"jsonnet": EngineTreeSitter,
-		"xlsg":    EngineTreeSitter,
+		"javascript": EngineTreeSitter,
+		"jsx":        EngineTreeSitter,
+		"typescript": EngineTreeSitter,
+		"tsx":        EngineTreeSitter,
+		"python":     EngineTreeSitter,
+		"java":       EngineTreeSitter,
+		"c":          EngineTreeSitter,
+		"cpp":        EngineTreeSitter,
+		"c++":        EngineTreeSitter,
+		"scala":      EngineTreeSitter,
+		"rust":       EngineTreeSitter,
+		"ruby":       EngineTreeSitter,
+		"c#":         EngineTreeSitter,
+		"jsonnet":    EngineTreeSitter,
+		"xlsg":       EngineTreeSitter,
 	},
 }
 

docker-images/syntax-highlighter/crates/sg-syntax/Cargo.toml

@@ -49,6 +49,12 @@ tree-sitter-scala = { git = "https://github.com/sourcegraph/tree-sitter-scala" }
 
 tree-sitter-xlsg = { git = "https://github.com/sourcegraph/tree-sitter-xlsg" }
 
+tree-sitter-c = "0.20.2"
+tree-sitter-java = "0.20.0"
+tree-sitter-javascript = "0.20.0"
+tree-sitter-rust = "0.20.3"
+tree-sitter-typescript = "0.20.2"
+
 [dev-dependencies]
 insta = "1.11.0"
 pretty_assertions = "1.2.1"

docker-images/syntax-highlighter/crates/sg-syntax/queries/c/highlights.scm

@@ -0,0 +1,84 @@
+"--" @identifier.operator
+"-" @identifier.operator
+"-=" @identifier.operator
+"->" @identifier.operator
+"=" @identifier.operator
+"!=" @identifier.operator
+"*" @identifier.operator
+"&" @identifier.operator
+"&&" @identifier.operator
+"+" @identifier.operator
+"++" @identifier.operator
+"+=" @identifier.operator
+"<" @identifier.operator
+"==" @identifier.operator
+">" @identifier.operator
+"||" @identifier.operator
+"!" @identifier.operator
+
+; "." @delimiter
+; ";" @delimiter
+
+(string_literal) @string
+(system_lib_string) @string
+
+(null) @constant.null
+(number_literal) @number
+(char_literal) @character
+(true) @boolean
+(false) @boolean
+
+(call_expression
+  function: (identifier) @identifier.function)
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @identifier.function))
+(function_declarator
+  declarator: (identifier) @identifier.function)
+(preproc_function_def
+  name: (identifier) @identifier.function)
+
+(field_identifier) @identifier ;; TODO: something better
+(statement_identifier) @identifier
+(type_identifier) @type
+(primitive_type) @type.builtin
+(sized_type_specifier) @type
+
+((identifier) @constant
+ (#match? @constant "^[A-Z][A-Z\\d_]*$"))
+
+(identifier) @identifier
+
+(comment) @comment
+
+"break" @keyword
+"case" @keyword
+"const" @keyword
+"continue" @keyword
+"default" @keyword
+"do" @keyword
+"else" @keyword
+"enum" @keyword
+"extern" @keyword
+"for" @keyword
+"if" @keyword
+"inline" @keyword
+"return" @keyword
+"sizeof" @keyword
+"static" @keyword
+"struct" @keyword
+"switch" @keyword
+"typedef" @keyword
+"union" @keyword
+"volatile" @keyword
+"while" @keyword
+
+"#define" @keyword
+"#elif" @keyword
+"#else" @keyword
+"#endif" @keyword
+"#if" @keyword
+"#ifdef" @keyword
+"#ifndef" @keyword
+"#include" @keyword
+(preproc_directive) @keyword

docker-images/syntax-highlighter/crates/sg-syntax/queries/java/highlights.scm

@@ -0,0 +1,143 @@
+; Methods
+
+(method_declaration
+  name: (identifier) @identifier.function)
+(method_invocation
+  name: (identifier) @identifier.function)
+(super) @identifier.builtin
+
+; Annotations
+
+(annotation
+  name: (identifier) @identifier.attribute)
+(marker_annotation
+  name: (identifier) @identifier.attribute)
+
+"@" @operator
+
+; Types
+
+(type_identifier) @identifier.type
+
+(interface_declaration
+  name: (identifier) @identifier.type)
+(class_declaration
+  name: (identifier) @identifier.type)
+(enum_declaration
+  name: (identifier) @identifier.type)
+
+((field_access
+  object: (identifier) @identifier.type)
+ (#match? @identifier.type "^[A-Z]"))
+((scoped_identifier
+  scope: (identifier) @identifier.type)
+ (#match? @identifier.type "^[A-Z]"))
+((method_invocation
+  object: (identifier) @identifier.type)
+ (#match? @identifier.type "^[A-Z]"))
+((method_reference
+  . (identifier) @identifier.type)
+ (#match? @identifier.type "^[A-Z]"))
+
+(constructor_declaration
+  name: (identifier) @identifier.type)
+
+[
+  (boolean_type)
+  (integral_type)
+  (floating_point_type)
+  (floating_point_type)
+  (void_type)
+] @identifier.builtin
+
+; Variables
+
+((identifier) @constant
+ (#match? @constant "^_*[A-Z][A-Z\\d_]+$"))
+
+(identifier) @identifier
+
+(this) @identifier.builtin
+
+; Literals
+
+[
+  (hex_integer_literal)
+  (decimal_integer_literal)
+  (octal_integer_literal)
+  (decimal_floating_point_literal)
+  (hex_floating_point_literal)
+] @number
+
+[
+  (character_literal)
+  (string_literal)
+  (text_block)
+] @string
+
+[
+  (true)
+  (false)
+] @boolean
+
+(null_literal) @constant.null
+
+[
+  (line_comment)
+  (block_comment)
+] @comment
+
+; Keywords
+
+[
+  "abstract"
+  "assert"
+  "break"
+  "case"
+  "catch"
+  "class"
+  "record"
+  "continue"
+  "default"
+  "do"
+  "else"
+  "enum"
+  "exports"
+  "extends"
+  "final"
+  "finally"
+  "for"
+  "if"
+  "implements"
+  "import"
+  "instanceof"
+  "interface"
+  "module"
+  "native"
+  "new"
+  "non-sealed"
+  "open"
+  "opens"
+  "package"
+  "private"
+  "protected"
+  "provides"
+  "public"
+  "requires"
+  "return"
+  "sealed"
+  "static"
+  "strictfp"
+  "switch"
+  "synchronized"
+  "throw"
+  "throws"
+  "to"
+  "transient"
+  "transitive"
+  "try"
+  "uses"
+  "volatile"
+  "while"
+  "with"
+] @keyword