Merge pull request #3782 from lydell/regex

Fix #3410, #3182: Allow regex to start with space or =

Author: michaelficarra

lib/coffee-script/lexer.js

@@ -258,10 +258,15 @@ exports.Lexer = class Lexer
       when @chunk[...3] is '///'
         {tokens, index} = @matchWithInterpolations @chunk[3..], HEREGEX, '///', 3
       when match = REGEX.exec @chunk
-        [regex] = match
+        [regex, closed] = match
         index = regex.length
         prev = last @tokens
-        return 0 if prev and (prev[0] in (if prev.spaced then NOT_REGEX else NOT_SPACED_REGEX))
+        if prev
+          if prev.spaced and prev[0] in CALLABLE
+            return 0 if not closed or POSSIBLY_DIVISION.test regex
+          else if prev[0] in NOT_REGEX
+            return 0
+        @error 'missing / (unclosed regex)' unless closed
       else
         return 0
 
@@ -776,13 +781,13 @@ HEREDOC_INDENT = /\n+([^\n\S]*)(?=\S)/g
 
 # Regex-matching-regexes.
 REGEX = /// ^
-  / (?! [\s=] ) (    # disallow leading whitespace or equals sign
+  / (?!/) (
   ?: [^ [ / \n \\ ]  # every other thing
    | \\.             # anything (but newlines) escaped
    | \[              # character class
        (?: \\. | [^ \] \n \\ ] )*
      ]
-  )+ /
+  )* (/)?
 ///
 
 REGEX_FLAGS  = /^\w*/
@@ -798,6 +803,8 @@ HEREGEX_OMIT = ///
 
 REGEX_ILLEGAL = /// ^ ( / | /{3}\s*) (\*) ///
 
+POSSIBLY_DIVISION   = /// ^ /=?\s ///
+
 # Other regexes.
 MULTILINER          = /\n/g
 
@@ -841,23 +848,17 @@ RELATION = ['IN', 'OF', 'INSTANCEOF']
 # Boolean tokens.
 BOOL = ['TRUE', 'FALSE']
 
-# Tokens which a regular expression will never immediately follow, but which
-# a division operator might.
-#
-# See: http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
-#
-# Our list is shorter, due to sans-parentheses method calls.
-NOT_REGEX = ['NUMBER', 'REGEX', 'BOOL', 'NULL', 'UNDEFINED', '++', '--']
-
-# If the previous token is not spaced, there are more preceding tokens that
-# force a division parse:
-NOT_SPACED_REGEX = NOT_REGEX.concat ')', '}', 'THIS', 'IDENTIFIER', 'STRING', ']'
-
 # Tokens which could legitimately be invoked or indexed. An opening
 # parentheses or bracket following these tokens will be recorded as the start
 # of a function invocation or indexing operation.
-CALLABLE  = ['IDENTIFIER', 'STRING', 'REGEX', ')', ']', '}', '?', '::', '@', 'THIS', 'SUPER']
-INDEXABLE = CALLABLE.concat 'NUMBER', 'BOOL', 'NULL', 'UNDEFINED'
+CALLABLE  = ['IDENTIFIER', ')', ']', '?', '@', 'THIS', 'SUPER']
+INDEXABLE = CALLABLE.concat ['NUMBER', 'STRING', 'REGEX', 'BOOL', 'NULL', 'UNDEFINED', '}', '::']
+
+# Tokens which a regular expression will never immediately follow (except spaced
+# CALLABLEs in some cases), but which a division operator can.
+#
+# See: http://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
+NOT_REGEX = INDEXABLE.concat ['++', '--']
 
 # Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
 # occurs at the start of a line. We disambiguate these from trailing whens to

src/lexer.coffee

@@ -405,3 +405,29 @@ test "missing `)`, `}`, `]`", ->
       foo#{ bar "#{1}"
           ^
   '''
+
+test "unclosed regexes", ->
+  assertErrorFormat '''
+    /
+  ''', '''
+    [stdin]:1:1: error: missing / (unclosed regex)
+    /
+    ^
+  '''
+  assertErrorFormat '''
+    # Note the double escaping; this would be `/a\/` real code.
+    /a\\/
+  ''', '''
+    [stdin]:2:1: error: missing / (unclosed regex)
+    /a\\/
+    ^
+  '''
+  assertErrorFormat '''
+    /// ^
+      a #{""" ""#{if /[/].test "|" then 1 else 0}"" """}
+    ///
+  ''', '''
+    [stdin]:2:18: error: missing / (unclosed regex)
+      a #{""" ""#{if /[/].test "|" then 1 else 0}"" """}
+                     ^
+  '''

test/error_messages.coffee

@@ -13,16 +13,34 @@ test "basic regular expression literals", ->
   ok 'a'.match /a/g
 
 test "division is not confused for a regular expression", ->
+  # Any spacing around the slash is allowed when it cannot be a regex.
   eq 2, 4 / 2 / 1
-
-  a = 4
+  eq 2, 4/2/1
+  eq 2, 4/ 2 / 1
+  eq 2, 4 /2 / 1
+  eq 2, 4 / 2/ 1
+  eq 2, 4 / 2 /1
+  eq 2, 4 /2/ 1
+
+  a = (regex) -> regex.test 'a b c'
+  a.valueOf = -> 4
   b = 2
   g = 1
-  eq 2, a / b/g
 
-  a = 10
-  b = a /= 4 / 2
-  eq a, 5
+  eq 2, a / b/g
+  eq 2, a/ b/g
+  eq 2, a / b/ g
+  eq 2, a	/	b/g # Tabs.
+  eq 2, a / b/g # Non-breaking spaces.
+  eq true, a /b/g
+  # Use parentheses to disambiguate.
+  eq true, a(/ b/g)
+  eq true, a(/ b/)
+  eq true, a (/ b/)
+  # Escape to disambiguate.
+  eq true, a /\ b/g
+  eq false, a	/\	b/g
+  eq true, a /\ b/
 
   obj = method: -> 2
   two = 2
@@ -32,6 +50,173 @@ test "division is not confused for a regular expression", ->
   eq 2, (4)/2/i
   eq 1, i/i/i
 
+  a = ''
+  a += ' ' until /   /.test a
+  eq a, '   '
+
+  a = if /=/.test '=' then yes else no
+  eq a, yes
+
+  a = if !/=/.test '=' then yes else no
+  eq a, no
+
+  #3182:
+  match = 'foo=bar'.match /=/
+  eq match[0], '='
+
+  #3410:
+  ok ' '.match(/ /)[0] is ' '
+
+
+test "division vs regex after a callable token", ->
+  b = 2
+  g = 1
+  r = (r) -> r.test 'b'
+
+  a = 4
+  eq 2, a / b/g
+  eq 2, a/b/g
+  eq 2, a/ b/g
+  eq true, r /b/g
+  eq 2, (1 + 3) / b/g
+  eq 2, (1 + 3)/b/g
+  eq 2, (1 + 3)/ b/g
+  eq true, (r) /b/g
+  eq 2, [4][0] / b/g
+  eq 2, [4][0]/b/g
+  eq 2, [4][0]/ b/g
+  eq true, [r][0] /b/g
+  eq 0.5, 4? / b/g
+  eq 0.5, 4?/b/g
+  eq 0.5, 4?/ b/g
+  eq true, r? /b/g
+  (->
+    eq 2, @ / b/g
+    eq 2, @/b/g
+    eq 2, @/ b/g
+  ).call 4
+  (->
+    eq true, @ /b/g
+  ).call r
+  (->
+    eq 2, this / b/g
+    eq 2, this/b/g
+    eq 2, this/ b/g
+  ).call 4
+  (->
+    eq true, this /b/g
+  ).call r
+  class A
+    p: (regex) -> if regex then r regex else 4
+  class B extends A
+    p: ->
+      eq 2, super / b/g
+      eq 2, super/b/g
+      eq 2, super/ b/g
+      eq true, super /b/g
+  new B().p()
+
+test "always division and never regex after some tokens", ->
+  b = 2
+  g = 1
+
+  eq 2, 4 / b/g
+  eq 2, 4/b/g
+  eq 2, 4/ b/g
+  eq 2, 4 /b/g
+  eq 2, "4" / b/g
+  eq 2, "4"/b/g
+  eq 2, "4"/ b/g
+  eq 2, "4" /b/g
+  ok isNaN /a/ / b/g
+  ok isNaN /a/i / b/g
+  ok isNaN /a//b/g
+  ok isNaN /a/i/b/g
+  ok isNaN /a// b/g
+  ok isNaN /a/i/ b/g
+  ok isNaN /a/ /b/g
+  ok isNaN /a/i /b/g
+  eq 0.5, true / b/g
+  eq 0.5, true/b/g
+  eq 0.5, true/ b/g
+  eq 0.5, true /b/g
+  eq 0, false / b/g
+  eq 0, false/b/g
+  eq 0, false/ b/g
+  eq 0, false /b/g
+  eq 0, null / b/g
+  eq 0, null/b/g
+  eq 0, null/ b/g
+  eq 0, null /b/g
+  ok isNaN undefined / b/g
+  ok isNaN undefined/b/g
+  ok isNaN undefined/ b/g
+  ok isNaN undefined /b/g
+  ok isNaN {a: 4} / b/g
+  ok isNaN {a: 4}/b/g
+  ok isNaN {a: 4}/ b/g
+  ok isNaN {a: 4} /b/g
+  o = prototype: 4
+  eq 2, o:: / b/g
+  eq 2, o::/b/g
+  eq 2, o::/ b/g
+  eq 2, o:: /b/g
+  i = 4
+  eq 2.0, i++ / b/g
+  eq 2.5, i++/b/g
+  eq 3.0, i++/ b/g
+  eq 3.5, i++ /b/g
+  eq 4.0, i-- / b/g
+  eq 3.5, i--/b/g
+  eq 3.0, i--/ b/g
+  eq 2.5, i-- /b/g
+
+test "compound division vs regex", ->
+  c = 4
+  i = 2
+
+  a = 10
+  b = a /= c / i
+  eq a, 5
+
+  a = 10
+  b = a /= c /i
+  eq a, 5
+
+  a = 10
+  b = a	/=	c /i # Tabs.
+  eq a, 5
+
+  a = 10
+  b = a /= c /i # Non-breaking spaces.
+  eq a, 5
+
+  a = 10
+  b = a/= c /i
+  eq a, 5
+
+  a = 10
+  b = a/=c/i
+  eq a, 5
+
+  a = (regex) -> regex.test '=C '
+  b = a /=c /i
+  eq b, true
+
+  a = (regex) -> regex.test '= C '
+  # Use parentheses to disambiguate.
+  b = a(/= c /i)
+  eq b, true
+  b = a(/= c /)
+  eq b, false
+  b = a (/= c /)
+  eq b, false
+  # Escape to disambiguate.
+  b = a /\= c /i
+  eq b, true
+  b = a /\= c /
+  eq b, false
+
 test "#764: regular expressions should be indexable", ->
   eq /0/['source'], ///#{0}///['source']