Support text appearing after "# type: ignore" (#116)

This is to allow things like `# type: ignore[E1000]`.
Essentially a backport of GH-13238, GH-13479, and GH-13504 from cpython.

Author: msullivan

.gitattributes

@@ -0,0 +1,16 @@
+# Generated files
+# https://github.com/github/linguist#generated-code
+ast3/Include/graminit.h          linguist-generated=true
+ast3/Python/graminit.h           linguist-generated=true
+ast3/Include/Python-ast.h        linguist-generated=true
+ast3/Python/Python-ast.c         linguist-generated=true
+ast3/Include/token.h             linguist-generated=true
+ast3/Lib/token.py                linguist-generated=true
+ast3/Parser/token.c              linguist-generated=true
+ast27/Include/graminit.h         linguist-generated=true
+ast27/Python/graminit.h          linguist-generated=true
+ast27/Include/Python-ast.h       linguist-generated=true
+ast27/Python/Python-ast.c        linguist-generated=true
+ast27/Include/token.h            linguist-generated=true
+ast27/Lib/token.py               linguist-generated=true
+ast27/Parser/token.c             linguist-generated=true

ast27/Include/Python-ast.h

@@ -10,7 +10,7 @@ module Python version "$Revision$"
 	    -- not really an actual node but useful in Jython's typesystem.
 	    | Suite(stmt* body)
 
-	stmt = FunctionDef(identifier name, arguments args, 
+	stmt = FunctionDef(identifier name, arguments args,
                             stmt* body, expr* decorator_list, string? type_comment)
 	      | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
 	      | Return(expr? value)
@@ -78,21 +78,21 @@ module Python version "$Revision$"
 	     | Attribute(expr value, identifier attr, expr_context ctx)
 	     | Subscript(expr value, slice slice, expr_context ctx)
 	     | Name(identifier id, expr_context ctx)
-	     | List(expr* elts, expr_context ctx) 
+	     | List(expr* elts, expr_context ctx)
 	     | Tuple(expr* elts, expr_context ctx)
 
 	      -- col_offset is the byte offset in the utf8 string the parser uses
 	      attributes (int lineno, int col_offset)
 
 	expr_context = Load | Store | Del | AugLoad | AugStore | Param
 
-	slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) 
-	      | ExtSlice(slice* dims) 
-	      | Index(expr value) 
+	slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step)
+	      | ExtSlice(slice* dims)
+	      | Index(expr value)
 
-	boolop = And | Or 
+	boolop = And | Or
 
-	operator = Add | Sub | Mult | Div | Mod | Pow | LShift 
+	operator = Add | Sub | Mult | Div | Mod | Pow | LShift
                  | RShift | BitOr | BitXor | BitAnd | FloorDiv
 
 	unaryop = Invert | Not | UAdd | USub
@@ -109,7 +109,7 @@ module Python version "$Revision$"
 	-- It is either an empty list or a list with length equal to the number of
 	-- args (including varargs and kwargs, if present) and with members set to the
 	-- string of each arg's type comment, if present, or None otherwise.
-	arguments = (expr* args, identifier? vararg, 
+	arguments = (expr* args, identifier? vararg,
 		     identifier? kwarg, expr* defaults, string* type_comments)
 
         -- keyword arguments supplied to call
@@ -118,5 +118,5 @@ module Python version "$Revision$"
         -- import name with optional 'as' alias.
         alias = (identifier name, identifier? asname)
 
-  type_ignore = TypeIgnore(int lineno)
+  type_ignore = TypeIgnore(int lineno, string tag)
 }

ast27/Parser/Python.asdl

@@ -152,12 +152,16 @@ warn(const char *msg, const char *filename, int lineno)
 
 
 typedef struct {
-    int *items;
+    struct {
+        int lineno;
+        char *comment;
+    } *items;
     size_t size;
     size_t num_items;
-} growable_int_array;
+} growable_comment_array;
 
-int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+static int
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
     assert(initial_size > 0);
     arr->items = malloc(initial_size * sizeof(*arr->items));
     arr->size = initial_size;
@@ -166,20 +170,28 @@ int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
     return arr->items != NULL;
 }
 
-int growable_int_array_add(growable_int_array *arr, int item) {
+static int
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
     if (arr->num_items >= arr->size) {
         arr->size *= 2;
         arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
-        if (!arr->items)
+        if (!arr->items) {
             return 0;
+        }
     }
 
-    arr->items[arr->num_items] = item;
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;
     arr->num_items++;
     return 1;
 }
 
-void growable_int_array_deallocate(growable_int_array *arr) {
+static void
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    unsigned i;
+    for (i = 0; i < arr->num_items; i++) {
+        PyObject_FREE(arr->items[i].comment);
+    }
     free(arr->items);
 }
 
@@ -195,8 +207,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     node *n;
     int started = 0;
 
-    growable_int_array type_ignores;
-    if (!growable_int_array_init(&type_ignores, 10)) {
+    growable_comment_array type_ignores;
+    if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         Ta27Tokenizer_Free(tok);
         return NULL;
@@ -264,7 +276,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             col_offset = -1;
 
         if (type == TYPE_IGNORE) {
-            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+            if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
                 err_ret->error = E_NOMEM;
                 break;
             }
@@ -297,15 +309,23 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             REQ(ch, ENDMARKER);
 
             for (i = 0; i < type_ignores.num_items; i++) {
-                Ta27Node_AddChild(ch, TYPE_IGNORE, NULL, type_ignores.items[i], 0);
+                int res = Ta27Node_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+                                            type_ignores.items[i].lineno, 0);
+                if (res != 0) {
+                    err_ret->error = res;
+                    Ta27Node_Free(n);
+                    n = NULL;
+                    break;
+                }
+                type_ignores.items[i].comment = NULL;
             }
         }
-        growable_int_array_deallocate(&type_ignores);
-
     }
     else
         n = NULL;
 
+    growable_comment_array_deallocate(&type_ignores);
+
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
     *flags = ps->p_flags;
 #endif

ast27/Parser/parsetok.c

@@ -1400,20 +1400,22 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
         /* This is a type comment if we matched all of type_comment_prefix. */
         if (!*prefix) {
             int is_type_ignore = 1;
+            const char *ignore_end = p + 6;
             tok_backup(tok, c);  /* don't eat the newline or EOF */
 
             type_start = p;
 
-            is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-            p += 6;
-            while (is_type_ignore && p < tok->cur) {
-              if (*p == '#')
-                  break;
-              is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-              p++;
-            }
+            /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
+             * or anything ASCII and non-alphanumeric. */
+            is_type_ignore = (
+                tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+                && !(tok->cur > ignore_end
+                     && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
             if (is_type_ignore) {
+                *p_start = (char *) ignore_end;
+                *p_end = tok->cur;
+
                 /* If this type ignore is the only thing on the line, consume the newline also. */
                 if (blankline) {
                     tok_nextc(tok);

ast27/Parser/tokenizer.c

@@ -297,7 +297,10 @@ Ta27AST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
                 goto error;
 
             for (i = 0; i < num; i++) {
-                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+                string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+                if (!type_comment)
+                    goto error;
+                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
                 if (!ti)
                     goto error;
                 asdl_seq_SET(type_ignores, i, ti);
@@ -2431,7 +2434,7 @@ ast_for_print_stmt(struct compiling *c, const node *n)
         dest = ast_for_expr(c, CHILD(n, 2));
         if (!dest)
             return NULL;
-            start = 4;
+        start = 4;
     }
     values_count = (NCH(n) + 1 - start) / 2;
     if (values_count) {

ast27/Python/Python-ast.c

@@ -130,5 +130,5 @@ module Python
 
     withitem = (expr context_expr, expr? optional_vars)
 
-    type_ignore = TypeIgnore(int lineno)
+    type_ignore = TypeIgnore(int lineno, string tag)
 }