make all regex matching identifiers unicode aware

elixir supports unicode identifiers since 1.5
Fixes #274

Author: lukaszsamson

lib/elixir_sense.ex

@@ -515,7 +515,7 @@ defmodule ElixirSense do
   # without parens.
   #
   # Note: This will be removed after refactoring the parser to
-  # allow unparseable nodes in the AST.
+  # allow unparsable nodes in the AST.
   defp maybe_fix_autocomple_on_cursor(%Metadata{error: nil} = meta, _, _, _) do
     meta
   end
@@ -537,11 +537,13 @@ defmodule ElixirSense do
 
     # Fix incomplete kw key, e.g. cursor after `option1: 1, opt`
     fix_incomplete_kw_key = fn text_before, text_after ->
-      if Regex.match?(~r/\,\s*[a-z][a-zA-Z0-9_]*$/, text_before) do
+      if Regex.match?(~r/\,\s*([\p{L}_][\p{L}\p{N}_@]*[?!]?)?$/, text_before) do
         text_before <> ": :__fake_value__" <> text_after
       end
     end
 
+    # TODO this may no longer be needed
+    # only fix_incomplete_call has some tests depending on it
     fixers = [
       fix_incomplete_call,
       fix_incomplete_kw,

lib/elixir_sense/core/metadata_builder.ex

@@ -776,7 +776,10 @@ defmodule ElixirSense.Core.MetadataBuilder do
   end
 
   defp pre({:@, meta_attr, [{name, meta, params}]}, state) when is_atom(name) do
-    if String.match?(Atom.to_string(name), ~r/^[a-zA-Z_].*/) do
+    name_string = Atom.to_string(name)
+
+    if String.match?(name_string, ~r/^[_\p{Ll}\p{Lo}][\p{L}\p{N}_]*[?!]?$/) and
+         not String.starts_with?(name_string, "__atom_elixir_marker_") do
       line = Keyword.fetch!(meta_attr, :line)
       column = Keyword.fetch!(meta_attr, :column)
 

lib/elixir_sense/core/parser.ex

@@ -246,7 +246,7 @@ defmodule ElixirSense.Core.Parser do
         |> Enum.join("\n")
 
       _ ->
-        if Regex.match?(~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, token) do
+        if Regex.match?(~r/^[\p{L}_][\p{L}\p{N}_@]*[?!]?$/, token) do
           remove_line(source, line)
         else
           replace_line_with_marker(source, line)

lib/elixir_sense/plugins/ecto.ex

@@ -79,7 +79,8 @@ defmodule ElixirSense.Plugins.Ecto do
              _ -> nil
            end),
          assoc_code <- Source.text_after(text_before, line, col),
-         [_, var] <- Regex.run(~r/^assoc\(\s*([a-z][a-zA-Z0-9_]*)\s*,/, assoc_code),
+         [_, var] <-
+           Regex.run(~r/^assoc\(\s*([_\p{Ll}\p{Lo}][\p{L}\p{N}_]*[?!]?)\s*,/, assoc_code),
          %{^var => %{type: type}} <- Query.extract_bindings(text_before, from_info, env, meta),
          true <- function_exported?(type, :__schema__, 1) do
       {:override, Query.find_assoc_suggestions(type, hint)}

lib/elixir_sense/plugins/ecto/query.ex

@@ -29,7 +29,7 @@ defmodule ElixirSense.Plugins.Ecto.Query do
 
   @join_opts [on: "A query expression or keyword list to filter the join."]
 
-  @var_r "[a-z][a-zA-Z0-9_]*"
+  @var_r "[_\p{Ll}\p{Lo}][\p{L}\p{N}_]*[?!]?"
   @mod_r "[A-Z][a-zA-Z0-9_\.]*"
   @binding_r "(#{@var_r}) in (#{@mod_r}|assoc\\(\\s*#{@var_r},\\s*\\:#{@var_r}\\s*\\))"
 

lib/elixir_sense/providers/suggestion/reducers/callbacks.ex

@@ -75,7 +75,7 @@ defmodule ElixirSense.Providers.Suggestion.Reducers.Callbacks do
     list = Enum.sort(list)
 
     cond do
-      Regex.match?(~r/\s(def|defmacro)\s+[a-z|_]*$/, text_before) ->
+      Regex.match?(~r/\s(def|defmacro)\s+([_\p{Ll}\p{Lo}][\p{L}\p{N}_]*[?!]?)?$/, text_before) ->
         {:halt, %{acc | result: list}}
 
       match?({_f, _a}, scope) ->

test/elixir_sense/suggestions_test.exs

@@ -824,7 +824,7 @@ defmodule ElixirSense.SuggestionsTest do
 
     defmodule MyLocalModule do
       @behaviour MyBehaviour
-      
+
       @impl true
       def flatten(list) do
         []
@@ -911,7 +911,7 @@ defmodule ElixirSense.SuggestionsTest do
 
     defmodule MyLocalModule do
       @behaviour MyBehaviour
-      
+
       @impl true
       defmacro flatten(list) do
         []
@@ -951,7 +951,7 @@ defmodule ElixirSense.SuggestionsTest do
     buffer = """
     defmodule MyLocalModule do
       @behaviour ElixirSenseExample.BehaviourWithMeta
-      
+
       @impl true
       def flatten(list) do
         []
@@ -989,7 +989,7 @@ defmodule ElixirSense.SuggestionsTest do
     buffer = """
     defmodule MyLocalModule do
       @behaviour :gen_statem
-      
+
       @impl true
       def init(list) do
         []
@@ -1029,7 +1029,7 @@ defmodule ElixirSense.SuggestionsTest do
     buffer = """
     defmodule MyLocalModule do
       @behaviour ElixirSenseExample.BehaviourWithMeta
-      
+
       @impl true
       defmacro bar(list) do
         []