From f3e6bffa6eecdf974822931cb5208cdccb4e3ab9 Mon Sep 17 00:00:00 2001
From: Victor Petrovykh <victor@magic.io>
Date: Wed, 10 Oct 2018 18:40:48 -0400
Subject: [PATCH] Scope `from`, `import`, and `as` the same way in imports.

Use `keyword.control.import.python` scope for `from`, `import`, and `as`
keywords when they appear as part of an import statement.
Use `keyword.control.flow.python` scope for `from` and `as` if they
appear elsewhere.

Issue: #123.
---
 grammars/MagicPython.cson            |  64 +++++--
 grammars/MagicPython.tmLanguage      |  82 +++++++--
 grammars/src/MagicPython.syntax.yaml |  46 +++--
 test/atom-spec/python-spec.js        | 261 ++++++++++++++++++++++++++-
 test/expressions/keywords.py         |   4 +-
 test/statements/from1.py             |  33 ++++
 test/statements/import1.py           |   4 +-
 test/statements/import6.py           |   2 +-
 test/statements/import7.py           |   2 +-
 test/statements/import8.py           |  48 +++++
 test/statements/import9.py           |  67 +++++++
 11 files changed, 557 insertions(+), 56 deletions(-)
 create mode 100644 test/statements/from1.py
 create mode 100644 test/statements/import8.py
 create mode 100644 test/statements/import9.py

diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
index c3f182d4..d04da619 100644
--- a/grammars/MagicPython.cson
+++ b/grammars/MagicPython.cson
@@ -259,12 +259,32 @@ repository:
         name: "storage.type.function.python"
         match: "\\b((async\\s+)?\\s*def)\\b"
       }
+      {
+        name: "keyword.control.flow.python"
+        comment: '''
+          if `as` is eventually followed by `:` or line continuation
+          it's probably control flow like:
+              with foo as bar, \\
+                   Foo as Bar:
+                try:
+                  do_stuff()
+                except Exception as e:
+                  pass
+          
+        '''
+        match: "\\b(?<!\\.)as\\b(?=.*[:\\\\])"
+      }
+      {
+        name: "keyword.control.import.python"
+        comment: "other legal use of `as` is in an import"
+        match: "\\b(?<!\\.)as\\b"
+      }
       {
         name: "keyword.control.flow.python"
         match: '''
           (?x)
             \\b(?<!\\.)(
-              as | async | continue | del | assert | break | finally | for
+              async | continue | del | assert | break | finally | for
               | from | elif | else | if | except | pass | raise
               | return | try | while | with
             )\\b
@@ -997,27 +1017,41 @@ repository:
     match: "(}(?!}))"
   import:
     comment: '''
-      Import statements
+      Import statements used to correctly mark `from`, `import`, and `as`
       
     '''
     patterns: [
       {
-        match: '''
-          (?x)
-            \\s* \\b(from) \\s*(\\.+)\\s* (import\\b)?
-          
-        '''
-        captures:
+        begin: "\\b(?<!\\.)(from)\\b(?=.+import)"
+        end: "$|(?=import)"
+        beginCaptures:
           "1":
             name: "keyword.control.import.python"
-          "2":
+        patterns: [
+          {
             name: "punctuation.separator.period.python"
-          "3":
-            name: "keyword.control.import.python"
+            match: "\\.+"
+          }
+          {
+            include: "#expression"
+          }
+        ]
       }
       {
-        name: "keyword.control.import.python"
-        match: "\\b(?<!\\.)import\\b"
+        begin: "\\b(?<!\\.)(import)\\b"
+        end: "$"
+        beginCaptures:
+          "1":
+            name: "keyword.control.import.python"
+        patterns: [
+          {
+            name: "keyword.control.import.python"
+            match: "\\b(?<!\\.)as\\b"
+          }
+          {
+            include: "#expression"
+          }
+        ]
       }
     ]
   "class-declaration":
@@ -1837,13 +1871,13 @@ repository:
       (?x)
         \\b(?:
           (
-            and | as | assert | async | await | break | class | continue | def
+            and | assert | async | await | break | class | continue | def
             | del | elif | else | except | finally | for | from | global
             | if | in | is | (?<=\\.)lambda | lambda(?=\\s*[\\.=])
             | nonlocal | not | or | pass | raise | return | try | while | with
             | yield
           ) | (
-            import
+            as | import
           )
         )\\b
       
diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
index 5a92c63d..7a833786 100644
--- a/grammars/MagicPython.tmLanguage
+++ b/grammars/MagicPython.tmLanguage
@@ -416,13 +416,37 @@
             <key>match</key>
             <string>\b((async\s+)?\s*def)\b</string>
           </dict>
+          <dict>
+            <key>name</key>
+            <string>keyword.control.flow.python</string>
+            <key>comment</key>
+            <string>if `as` is eventually followed by `:` or line continuation
+it's probably control flow like:
+    with foo as bar, \
+         Foo as Bar:
+      try:
+        do_stuff()
+      except Exception as e:
+        pass
+</string>
+            <key>match</key>
+            <string>\b(?&lt;!\.)as\b(?=.*[:\\])</string>
+          </dict>
+          <dict>
+            <key>name</key>
+            <string>keyword.control.import.python</string>
+            <key>comment</key>
+            <string>other legal use of `as` is in an import</string>
+            <key>match</key>
+            <string>\b(?&lt;!\.)as\b</string>
+          </dict>
           <dict>
             <key>name</key>
             <string>keyword.control.flow.python</string>
             <key>match</key>
             <string>(?x)
   \b(?&lt;!\.)(
-    as | async | continue | del | assert | break | finally | for
+    async | continue | del | assert | break | finally | for
     | from | elif | else | if | except | pass | raise
     | return | try | while | with
   )\b
@@ -1531,39 +1555,63 @@
       <key>import</key>
       <dict>
         <key>comment</key>
-        <string>Import statements
+        <string>Import statements used to correctly mark `from`, `import`, and `as`
 </string>
         <key>patterns</key>
         <array>
           <dict>
-            <key>match</key>
-            <string>(?x)
-  \s* \b(from) \s*(\.+)\s* (import\b)?
-</string>
-            <key>captures</key>
+            <key>begin</key>
+            <string>\b(?&lt;!\.)(from)\b(?=.+import)</string>
+            <key>end</key>
+            <string>$|(?=import)</string>
+            <key>beginCaptures</key>
             <dict>
               <key>1</key>
               <dict>
                 <key>name</key>
                 <string>keyword.control.import.python</string>
               </dict>
-              <key>2</key>
+            </dict>
+            <key>patterns</key>
+            <array>
               <dict>
                 <key>name</key>
                 <string>punctuation.separator.period.python</string>
+                <key>match</key>
+                <string>\.+</string>
               </dict>
-              <key>3</key>
+              <dict>
+                <key>include</key>
+                <string>#expression</string>
+              </dict>
+            </array>
+          </dict>
+          <dict>
+            <key>begin</key>
+            <string>\b(?&lt;!\.)(import)\b</string>
+            <key>end</key>
+            <string>$</string>
+            <key>beginCaptures</key>
+            <dict>
+              <key>1</key>
               <dict>
                 <key>name</key>
                 <string>keyword.control.import.python</string>
               </dict>
             </dict>
-          </dict>
-          <dict>
-            <key>name</key>
-            <string>keyword.control.import.python</string>
-            <key>match</key>
-            <string>\b(?&lt;!\.)import\b</string>
+            <key>patterns</key>
+            <array>
+              <dict>
+                <key>name</key>
+                <string>keyword.control.import.python</string>
+                <key>match</key>
+                <string>\b(?&lt;!\.)as\b</string>
+              </dict>
+              <dict>
+                <key>include</key>
+                <string>#expression</string>
+              </dict>
+            </array>
           </dict>
         </array>
       </dict>
@@ -2836,13 +2884,13 @@ indirectly through syntactic constructs
         <string>(?x)
   \b(?:
     (
-      and | as | assert | async | await | break | class | continue | def
+      and | assert | async | await | break | class | continue | def
       | del | elif | else | except | finally | for | from | global
       | if | in | is | (?&lt;=\.)lambda | lambda(?=\s*[\.=])
       | nonlocal | not | or | pass | raise | return | try | while | with
       | yield
     ) | (
-      import
+      as | import
     )
   )\b
 </string>
diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml
index 2be48ca1..e7623f32 100644
--- a/grammars/src/MagicPython.syntax.yaml
+++ b/grammars/src/MagicPython.syntax.yaml
@@ -359,11 +359,25 @@ repository:
     patterns:
       - name: storage.type.function.python
         match: \b((async\s+)?\s*def)\b
+      - name: keyword.control.flow.python
+        comment: |
+          if `as` is eventually followed by `:` or line continuation
+          it's probably control flow like:
+              with foo as bar, \
+                   Foo as Bar:
+                try:
+                  do_stuff()
+                except Exception as e:
+                  pass
+        match: \b(?<!\.)as\b(?=.*[:\\])
+      - name: keyword.control.import.python
+        comment: other legal use of `as` is in an import
+        match: \b(?<!\.)as\b
       - name: keyword.control.flow.python
         match: |
           (?x)
             \b(?<!\.)(
-              as | async | continue | del | assert | break | finally | for
+              async | continue | del | assert | break | finally | for
               | from | elif | else | if | except | pass | raise
               | return | try | while | with
             )\b
@@ -842,17 +856,25 @@ repository:
 
   import:
     comment: |
-      Import statements
+      Import statements used to correctly mark `from`, `import`, and `as`
     patterns:
-      - match: |
-          (?x)
-            \s* \b(from) \s*(\.+)\s* (import\b)?
-        captures:
+      - begin: \b(?<!\.)(from)\b(?=.+import)
+        end: $|(?=import)
+        beginCaptures:
           '1': {name: keyword.control.import.python}
-          '2': {name: punctuation.separator.period.python}
-          '3': {name: keyword.control.import.python}
-      - name: keyword.control.import.python
-        match: \b(?<!\.)import\b
+        patterns:
+          - name: punctuation.separator.period.python
+            match: \.+
+          - include: '#expression'
+
+      - begin: \b(?<!\.)(import)\b
+        end: $
+        beginCaptures:
+          '1': {name: keyword.control.import.python}
+        patterns:
+          - name: keyword.control.import.python
+            match: \b(?<!\.)as\b
+          - include: '#expression'
 
   class-declaration:
     patterns:
@@ -1385,13 +1407,13 @@ repository:
       (?x)
         \b(?:
           (
-            and | as | assert | async | await | break | class | continue | def
+            and | assert | async | await | break | class | continue | def
             | del | elif | else | except | finally | for | from | global
             | if | in | is | (?<=\.)lambda | lambda(?=\s*[\.=])
             | nonlocal | not | or | pass | raise | return | try | while | with
             | yield
           ) | (
-            import
+            as | import
           )
         )\b
     captures:
diff --git a/test/atom-spec/python-spec.js b/test/atom-spec/python-spec.js
index 270b752b..37e92940 100644
--- a/test/atom-spec/python-spec.js
+++ b/test/atom-spec/python-spec.js
@@ -6078,7 +6078,7 @@ describe("Grammar Tests", function() {
     function() {
       tokens = grammar.tokenizeLines("as async await continue del assert break finally for\nfrom elif else if import except pass raise\nreturn try while with\n\nnonlocal global class def")
       expect(tokens[0][0].value).toBe("as");
-      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[0][1].value).toBe(" ");
       expect(tokens[0][1].scopes).toEqual(["source.python"]);
       expect(tokens[0][2].value).toBe("async");
@@ -6112,7 +6112,7 @@ describe("Grammar Tests", function() {
       expect(tokens[0][16].value).toBe("for");
       expect(tokens[0][16].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[1][0].value).toBe("from");
-      expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[1][1].value).toBe(" ");
       expect(tokens[1][1].scopes).toEqual(["source.python"]);
       expect(tokens[1][2].value).toBe("elif");
@@ -12119,6 +12119,63 @@ describe("Grammar Tests", function() {
       expect(tokens[3][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]);
     });
 
+  it("test/statements/from1.py", 
+    function() {
+      tokens = grammar.tokenizeLines("from ... import foo as bar\nraise Exception('done') from exc\nyield from foo")
+      expect(tokens[0][0].value).toBe("from");
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[0][1].value).toBe(" ");
+      expect(tokens[0][1].scopes).toEqual(["source.python"]);
+      expect(tokens[0][2].value).toBe("...");
+      expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]);
+      expect(tokens[0][3].value).toBe(" ");
+      expect(tokens[0][3].scopes).toEqual(["source.python"]);
+      expect(tokens[0][4].value).toBe("import");
+      expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[0][5].value).toBe(" ");
+      expect(tokens[0][5].scopes).toEqual(["source.python"]);
+      expect(tokens[0][6].value).toBe("foo");
+      expect(tokens[0][6].scopes).toEqual(["source.python"]);
+      expect(tokens[0][7].value).toBe(" ");
+      expect(tokens[0][7].scopes).toEqual(["source.python"]);
+      expect(tokens[0][8].value).toBe("as");
+      expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[0][9].value).toBe(" ");
+      expect(tokens[0][9].scopes).toEqual(["source.python"]);
+      expect(tokens[0][10].value).toBe("bar");
+      expect(tokens[0][10].scopes).toEqual(["source.python"]);
+      expect(tokens[1][0].value).toBe("raise");
+      expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[1][1].value).toBe(" ");
+      expect(tokens[1][1].scopes).toEqual(["source.python"]);
+      expect(tokens[1][2].value).toBe("Exception");
+      expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]);
+      expect(tokens[1][3].value).toBe("(");
+      expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]);
+      expect(tokens[1][4].value).toBe("'");
+      expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[1][5].value).toBe("done");
+      expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]);
+      expect(tokens[1][6].value).toBe("'");
+      expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[1][7].value).toBe(")");
+      expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]);
+      expect(tokens[1][8].value).toBe(" ");
+      expect(tokens[1][8].scopes).toEqual(["source.python"]);
+      expect(tokens[1][9].value).toBe("from");
+      expect(tokens[1][9].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[1][10].value).toBe(" ");
+      expect(tokens[1][10].scopes).toEqual(["source.python"]);
+      expect(tokens[1][11].value).toBe("exc");
+      expect(tokens[1][11].scopes).toEqual(["source.python"]);
+      expect(tokens[2][0].value).toBe("yield from");
+      expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[2][1].value).toBe(" ");
+      expect(tokens[2][1].scopes).toEqual(["source.python"]);
+      expect(tokens[2][2].value).toBe("foo");
+      expect(tokens[2][2].scopes).toEqual(["source.python"]);
+    });
+
   it("test/statements/global1.py", 
     function() {
       tokens = grammar.tokenizeLines("global a, b, c")
@@ -12237,7 +12294,7 @@ describe("Grammar Tests", function() {
       expect(tokens[0][8].value).toBe(" ");
       expect(tokens[0][8].scopes).toEqual(["source.python"]);
       expect(tokens[0][9].value).toBe("as");
-      expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[0][10].value).toBe(" ");
       expect(tokens[0][10].scopes).toEqual(["source.python"]);
       expect(tokens[0][11].value).toBe("spam");
@@ -12257,7 +12314,7 @@ describe("Grammar Tests", function() {
       expect(tokens[1][3].value).toBe(" ");
       expect(tokens[1][3].scopes).toEqual(["source.python"]);
       expect(tokens[1][4].value).toBe("as");
-      expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[1][5].value).toBe(" ");
       expect(tokens[1][5].scopes).toEqual(["source.python"]);
       expect(tokens[1][6].value).toBe("ham");
@@ -12502,7 +12559,7 @@ describe("Grammar Tests", function() {
     function() {
       tokens = grammar.tokenizeLines("from __future__ import generator_stop")
       expect(tokens[0][0].value).toBe("from");
-      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[0][1].value).toBe(" ");
       expect(tokens[0][1].scopes).toEqual(["source.python"]);
       expect(tokens[0][2].value).toBe("__future__");
@@ -12537,7 +12594,7 @@ describe("Grammar Tests", function() {
       expect(tokens[0][7].value).toBe("*");
       expect(tokens[0][7].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]);
       expect(tokens[1][0].value).toBe("from");
-      expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[1][1].value).toBe(" ");
       expect(tokens[1][1].scopes).toEqual(["source.python"]);
       expect(tokens[1][2].value).toBe("importing");
@@ -12552,6 +12609,198 @@ describe("Grammar Tests", function() {
       expect(tokens[1][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]);
     });
 
+  it("test/statements/import8.py", 
+    function() {
+      tokens = grammar.tokenizeLines("try:\n    import \\\n                    time as ham, \\\n                    datetime \\\n            # XXX: comment at the end of import\nexcept Exception as exc:\n    pass")
+      expect(tokens[0][0].value).toBe("try");
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][1].value).toBe(":");
+      expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]);
+      expect(tokens[1][0].value).toBe("    ");
+      expect(tokens[1][0].scopes).toEqual(["source.python"]);
+      expect(tokens[1][1].value).toBe("import");
+      expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[1][2].value).toBe(" ");
+      expect(tokens[1][2].scopes).toEqual(["source.python"]);
+      expect(tokens[1][3].value).toBe("\\");
+      expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]);
+      expect(tokens[1][4].value).toBe("");
+      expect(tokens[1][4].scopes).toEqual(["source.python"]);
+      expect(tokens[2][0].value).toBe("                    ");
+      expect(tokens[2][0].scopes).toEqual(["source.python"]);
+      expect(tokens[2][1].value).toBe("time");
+      expect(tokens[2][1].scopes).toEqual(["source.python"]);
+      expect(tokens[2][2].value).toBe(" ");
+      expect(tokens[2][2].scopes).toEqual(["source.python"]);
+      expect(tokens[2][3].value).toBe("as");
+      expect(tokens[2][3].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[2][4].value).toBe(" ");
+      expect(tokens[2][4].scopes).toEqual(["source.python"]);
+      expect(tokens[2][5].value).toBe("ham");
+      expect(tokens[2][5].scopes).toEqual(["source.python"]);
+      expect(tokens[2][6].value).toBe(",");
+      expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[2][7].value).toBe(" ");
+      expect(tokens[2][7].scopes).toEqual(["source.python"]);
+      expect(tokens[2][8].value).toBe("\\");
+      expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]);
+      expect(tokens[2][9].value).toBe("");
+      expect(tokens[2][9].scopes).toEqual(["source.python"]);
+      expect(tokens[3][0].value).toBe("                    ");
+      expect(tokens[3][0].scopes).toEqual(["source.python"]);
+      expect(tokens[3][1].value).toBe("datetime");
+      expect(tokens[3][1].scopes).toEqual(["source.python"]);
+      expect(tokens[3][2].value).toBe(" ");
+      expect(tokens[3][2].scopes).toEqual(["source.python"]);
+      expect(tokens[3][3].value).toBe("\\");
+      expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]);
+      expect(tokens[3][4].value).toBe("");
+      expect(tokens[3][4].scopes).toEqual(["source.python"]);
+      expect(tokens[4][0].value).toBe("            ");
+      expect(tokens[4][0].scopes).toEqual(["source.python"]);
+      expect(tokens[4][1].value).toBe("#");
+      expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
+      expect(tokens[4][2].value).toBe(" ");
+      expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[4][3].value).toBe("XXX");
+      expect(tokens[4][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]);
+      expect(tokens[4][4].value).toBe(": comment at the end of import");
+      expect(tokens[4][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[5][0].value).toBe("except");
+      expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[5][1].value).toBe(" ");
+      expect(tokens[5][1].scopes).toEqual(["source.python"]);
+      expect(tokens[5][2].value).toBe("Exception");
+      expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]);
+      expect(tokens[5][3].value).toBe(" ");
+      expect(tokens[5][3].scopes).toEqual(["source.python"]);
+      expect(tokens[5][4].value).toBe("as");
+      expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[5][5].value).toBe(" ");
+      expect(tokens[5][5].scopes).toEqual(["source.python"]);
+      expect(tokens[5][6].value).toBe("exc");
+      expect(tokens[5][6].scopes).toEqual(["source.python"]);
+      expect(tokens[5][7].value).toBe(":");
+      expect(tokens[5][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]);
+      expect(tokens[6][0].value).toBe("    ");
+      expect(tokens[6][0].scopes).toEqual(["source.python"]);
+      expect(tokens[6][1].value).toBe("pass");
+      expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+    });
+
+  it("test/statements/import9.py", 
+    function() {
+      tokens = grammar.tokenizeLines("from . . . foo import \\\n    (\n        # XXX: legal comment inside import\n        time as bar,\n        # another comment\n        baz,\n        datetime as ham\n    )\nraise Exception('!') from None")
+      expect(tokens[0][0].value).toBe("from");
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[0][1].value).toBe(" ");
+      expect(tokens[0][1].scopes).toEqual(["source.python"]);
+      expect(tokens[0][2].value).toBe(".");
+      expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]);
+      expect(tokens[0][3].value).toBe(" ");
+      expect(tokens[0][3].scopes).toEqual(["source.python"]);
+      expect(tokens[0][4].value).toBe(".");
+      expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.period.python"]);
+      expect(tokens[0][5].value).toBe(" ");
+      expect(tokens[0][5].scopes).toEqual(["source.python"]);
+      expect(tokens[0][6].value).toBe(".");
+      expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.period.python"]);
+      expect(tokens[0][7].value).toBe(" ");
+      expect(tokens[0][7].scopes).toEqual(["source.python"]);
+      expect(tokens[0][8].value).toBe("foo");
+      expect(tokens[0][8].scopes).toEqual(["source.python"]);
+      expect(tokens[0][9].value).toBe(" ");
+      expect(tokens[0][9].scopes).toEqual(["source.python"]);
+      expect(tokens[0][10].value).toBe("import");
+      expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[0][11].value).toBe(" ");
+      expect(tokens[0][11].scopes).toEqual(["source.python"]);
+      expect(tokens[0][12].value).toBe("\\");
+      expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]);
+      expect(tokens[0][13].value).toBe("");
+      expect(tokens[0][13].scopes).toEqual(["source.python"]);
+      expect(tokens[1][0].value).toBe("    ");
+      expect(tokens[1][0].scopes).toEqual(["source.python"]);
+      expect(tokens[1][1].value).toBe("(");
+      expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]);
+      expect(tokens[2][0].value).toBe("        ");
+      expect(tokens[2][0].scopes).toEqual(["source.python"]);
+      expect(tokens[2][1].value).toBe("#");
+      expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
+      expect(tokens[2][2].value).toBe(" ");
+      expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[2][3].value).toBe("XXX");
+      expect(tokens[2][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]);
+      expect(tokens[2][4].value).toBe(": legal comment inside import");
+      expect(tokens[2][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[3][0].value).toBe("        ");
+      expect(tokens[3][0].scopes).toEqual(["source.python"]);
+      expect(tokens[3][1].value).toBe("time");
+      expect(tokens[3][1].scopes).toEqual(["source.python"]);
+      expect(tokens[3][2].value).toBe(" ");
+      expect(tokens[3][2].scopes).toEqual(["source.python"]);
+      expect(tokens[3][3].value).toBe("as");
+      expect(tokens[3][3].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[3][4].value).toBe(" ");
+      expect(tokens[3][4].scopes).toEqual(["source.python"]);
+      expect(tokens[3][5].value).toBe("bar");
+      expect(tokens[3][5].scopes).toEqual(["source.python"]);
+      expect(tokens[3][6].value).toBe(",");
+      expect(tokens[3][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[4][0].value).toBe("        ");
+      expect(tokens[4][0].scopes).toEqual(["source.python"]);
+      expect(tokens[4][1].value).toBe("#");
+      expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
+      expect(tokens[4][2].value).toBe(" another comment");
+      expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[5][0].value).toBe("        ");
+      expect(tokens[5][0].scopes).toEqual(["source.python"]);
+      expect(tokens[5][1].value).toBe("baz");
+      expect(tokens[5][1].scopes).toEqual(["source.python"]);
+      expect(tokens[5][2].value).toBe(",");
+      expect(tokens[5][2].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[6][0].value).toBe("        ");
+      expect(tokens[6][0].scopes).toEqual(["source.python"]);
+      expect(tokens[6][1].value).toBe("datetime");
+      expect(tokens[6][1].scopes).toEqual(["source.python"]);
+      expect(tokens[6][2].value).toBe(" ");
+      expect(tokens[6][2].scopes).toEqual(["source.python"]);
+      expect(tokens[6][3].value).toBe("as");
+      expect(tokens[6][3].scopes).toEqual(["source.python","keyword.control.import.python"]);
+      expect(tokens[6][4].value).toBe(" ");
+      expect(tokens[6][4].scopes).toEqual(["source.python"]);
+      expect(tokens[6][5].value).toBe("ham");
+      expect(tokens[6][5].scopes).toEqual(["source.python"]);
+      expect(tokens[7][0].value).toBe("    ");
+      expect(tokens[7][0].scopes).toEqual(["source.python"]);
+      expect(tokens[7][1].value).toBe(")");
+      expect(tokens[7][1].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]);
+      expect(tokens[8][0].value).toBe("raise");
+      expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[8][1].value).toBe(" ");
+      expect(tokens[8][1].scopes).toEqual(["source.python"]);
+      expect(tokens[8][2].value).toBe("Exception");
+      expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]);
+      expect(tokens[8][3].value).toBe("(");
+      expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]);
+      expect(tokens[8][4].value).toBe("'");
+      expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[8][5].value).toBe("!");
+      expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]);
+      expect(tokens[8][6].value).toBe("'");
+      expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[8][7].value).toBe(")");
+      expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]);
+      expect(tokens[8][8].value).toBe(" ");
+      expect(tokens[8][8].scopes).toEqual(["source.python"]);
+      expect(tokens[8][9].value).toBe("from");
+      expect(tokens[8][9].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[8][10].value).toBe(" ");
+      expect(tokens[8][10].scopes).toEqual(["source.python"]);
+      expect(tokens[8][11].value).toBe("None");
+      expect(tokens[8][11].scopes).toEqual(["source.python","constant.language.python"]);
+    });
+
   it("test/statements/nonlocal1.py", 
     function() {
       tokens = grammar.tokenizeLines("nonlocal a, b, c")
diff --git a/test/expressions/keywords.py b/test/expressions/keywords.py
index 82ae6521..6ff1d516 100644
--- a/test/expressions/keywords.py
+++ b/test/expressions/keywords.py
@@ -6,7 +6,7 @@
 
 
 
-as            : keyword.control.flow.python, source.python
+as            : keyword.control.import.python, source.python
               : source.python
 async         : keyword.control.flow.python, source.python
               : source.python
@@ -23,7 +23,7 @@
 finally       : keyword.control.flow.python, source.python
               : source.python
 for           : keyword.control.flow.python, source.python
-from          : keyword.control.flow.python, source.python
+from          : keyword.control.import.python, source.python
               : source.python
 elif          : keyword.control.flow.python, source.python
               : source.python
diff --git a/test/statements/from1.py b/test/statements/from1.py
new file mode 100644
index 00000000..99d3a670
--- /dev/null
+++ b/test/statements/from1.py
@@ -0,0 +1,33 @@
+from ... import foo as bar
+raise Exception('done') from exc
+yield from foo
+
+
+
+
+from          : keyword.control.import.python, source.python
+              : source.python
+...           : punctuation.separator.period.python, source.python
+              : source.python
+import        : keyword.control.import.python, source.python
+              : source.python
+foo           : source.python
+              : source.python
+as            : keyword.control.import.python, source.python
+              : source.python
+bar           : source.python
+raise         : keyword.control.flow.python, source.python
+              : source.python
+Exception     : meta.function-call.python, source.python, support.type.exception.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+done          : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python
+)             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
+              : source.python
+from          : keyword.control.flow.python, source.python
+              : source.python
+exc           : source.python
+yield from    : keyword.control.flow.python, source.python
+              : source.python
+foo           : source.python
diff --git a/test/statements/import1.py b/test/statements/import1.py
index f41b72c5..6faca8b3 100644
--- a/test/statements/import1.py
+++ b/test/statements/import1.py
@@ -12,7 +12,7 @@
               : source.python
 bar           : source.python
               : source.python
-as            : keyword.control.flow.python, source.python
+as            : keyword.control.import.python, source.python
               : source.python
 spam          : source.python
 ,             : punctuation.separator.element.python, source.python
@@ -22,7 +22,7 @@
               : source.python
 time          : source.python
               : source.python
-as            : keyword.control.flow.python, source.python
+as            : keyword.control.import.python, source.python
               : source.python
 ham           : source.python
 ,             : punctuation.separator.element.python, source.python
diff --git a/test/statements/import6.py b/test/statements/import6.py
index 18bacd0e..a39a790e 100644
--- a/test/statements/import6.py
+++ b/test/statements/import6.py
@@ -2,7 +2,7 @@
 
 
 
-from          : keyword.control.flow.python, source.python
+from          : keyword.control.import.python, source.python
               : source.python
 __future__    : source.python, support.variable.magic.python
               : source.python
diff --git a/test/statements/import7.py b/test/statements/import7.py
index 7c0fd75f..7951802b 100644
--- a/test/statements/import7.py
+++ b/test/statements/import7.py
@@ -11,7 +11,7 @@
 import        : keyword.control.import.python, source.python
               : source.python
 *             : keyword.operator.arithmetic.python, source.python
-from          : keyword.control.flow.python, source.python
+from          : keyword.control.import.python, source.python
               : source.python
 importing     : source.python
               : source.python
diff --git a/test/statements/import8.py b/test/statements/import8.py
new file mode 100644
index 00000000..1d49bbd1
--- /dev/null
+++ b/test/statements/import8.py
@@ -0,0 +1,48 @@
+try:
+    import \
+                    time as ham, \
+                    datetime \
+            # XXX: comment at the end of import
+except Exception as exc:
+    pass
+
+
+
+
+try           : keyword.control.flow.python, source.python
+:             : punctuation.separator.colon.python, source.python
+              : source.python
+import        : keyword.control.import.python, source.python
+              : source.python
+\             : punctuation.separator.continuation.line.python, source.python
+              : source.python
+                     : source.python
+time          : source.python
+              : source.python
+as            : keyword.control.import.python, source.python
+              : source.python
+ham           : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+\             : punctuation.separator.continuation.line.python, source.python
+              : source.python
+                     : source.python
+datetime      : source.python
+              : source.python
+\             : punctuation.separator.continuation.line.python, source.python
+              : source.python
+              : source.python
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+              : comment.line.number-sign.python, source.python
+XXX           : comment.line.number-sign.python, keyword.codetag.notation.python, source.python
+: comment at the end of import : comment.line.number-sign.python, source.python
+except        : keyword.control.flow.python, source.python
+              : source.python
+Exception     : source.python, support.type.exception.python
+              : source.python
+as            : keyword.control.flow.python, source.python
+              : source.python
+exc           : source.python
+:             : punctuation.separator.colon.python, source.python
+              : source.python
+pass          : keyword.control.flow.python, source.python
diff --git a/test/statements/import9.py b/test/statements/import9.py
new file mode 100644
index 00000000..2fa6bfac
--- /dev/null
+++ b/test/statements/import9.py
@@ -0,0 +1,67 @@
+from . . . foo import \
+    (
+        # XXX: legal comment inside import
+        time as bar,
+        # another comment
+        baz,
+        datetime as ham
+    )
+raise Exception('!') from None
+
+
+
+
+from          : keyword.control.import.python, source.python
+              : source.python
+.             : punctuation.separator.period.python, source.python
+              : source.python
+.             : punctuation.separator.period.python, source.python
+              : source.python
+.             : punctuation.separator.period.python, source.python
+              : source.python
+foo           : source.python
+              : source.python
+import        : keyword.control.import.python, source.python
+              : source.python
+\             : punctuation.separator.continuation.line.python, source.python
+              : source.python
+              : source.python
+(             : punctuation.parenthesis.begin.python, source.python
+              : source.python
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+              : comment.line.number-sign.python, source.python
+XXX           : comment.line.number-sign.python, keyword.codetag.notation.python, source.python
+: legal comment inside import : comment.line.number-sign.python, source.python
+              : source.python
+time          : source.python
+              : source.python
+as            : keyword.control.import.python, source.python
+              : source.python
+bar           : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+ another comment : comment.line.number-sign.python, source.python
+              : source.python
+baz           : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+datetime      : source.python
+              : source.python
+as            : keyword.control.import.python, source.python
+              : source.python
+ham           : source.python
+              : source.python
+)             : punctuation.parenthesis.end.python, source.python
+raise         : keyword.control.flow.python, source.python
+              : source.python
+Exception     : meta.function-call.python, source.python, support.type.exception.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+!             : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python
+'             : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python
+)             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
+              : source.python
+from          : keyword.control.flow.python, source.python
+              : source.python
+None          : constant.language.python, source.python