Skip to content

Commit f340032

Browse files
vpetrovykhvpetrovykh
committed
Fix a docstring highlighter bug.
1 parent 935abef commit f340032

12 files changed

+172
-20
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ci-test: release
1616
then echo "Error: package.version != git.tag" && exit 1 ; fi
1717

1818
test: ci-test
19-
atom -t .
19+
atom -t test/atom-spec
2020

2121
devenv:
2222
npm install [email protected]

‎grammars/MagicPython.cson

+6-5
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,8 @@ repository:
129129
]
130130
"docstring-statement":
131131
begin: "^(?=\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))"
132-
end: "(?<=\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\")"
132+
comment: "the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring"
133+
end: "((?<=\\1)|^)(?!\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))"
133134
patterns: [
134135
{
135136
include: "#docstring"
@@ -186,7 +187,7 @@ repository:
186187
{
187188
name: "string.quoted.docstring.single.python"
188189
begin: "(\\'|\\\")"
189-
end: "(\\1)|((?<!\\\\)\\n)"
190+
end: "(\\1)|(\\n)"
190191
beginCaptures:
191192
"1":
192193
name: "punctuation.definition.string.begin.python"
@@ -207,7 +208,7 @@ repository:
207208
{
208209
name: "string.quoted.docstring.raw.single.python"
209210
begin: "([rR])(\\'|\\\")"
210-
end: "(\\2)|((?<!\\\\)\\n)"
211+
end: "(\\2)|(\\n)"
211212
beginCaptures:
212213
"1":
213214
name: "storage.type.string.python"
@@ -4177,7 +4178,7 @@ repository:
41774178
]
41784179
"string-quoted-single-line":
41794180
name: "string.quoted.single.python"
4180-
begin: "(\\b[rR](?=[uU]))?([uU])?((['\"]))"
4181+
begin: "(?:\\b([rR])(?=[uU]))?([uU])?((['\"]))"
41814182
end: "(\\3)|((?<!\\\\)\\n)"
41824183
beginCaptures:
41834184
"1":
@@ -4351,7 +4352,7 @@ repository:
43514352
]
43524353
"string-quoted-multi-line":
43534354
name: "string.quoted.multi.python"
4354-
begin: "(\\b[rR](?=[uU]))?([uU])?('''|\"\"\")"
4355+
begin: "(?:\\b([rR])(?=[uU]))?([uU])?('''|\"\"\")"
43554356
end: "(\\3)"
43564357
beginCaptures:
43574358
"1":

‎grammars/MagicPython.tmLanguage

+7-5
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,10 @@
184184
<dict>
185185
<key>begin</key>
186186
<string>^(?=\s*[rR]?(\'\'\'|\"\"\"|\'|\"))</string>
187+
<key>comment</key>
188+
<string>the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring</string>
187189
<key>end</key>
188-
<string>(?&lt;=\'\'\'|\"\"\"|\'|\")</string>
190+
<string>((?&lt;=\1)|^)(?!\s*[rR]?(\'\'\'|\"\"\"|\'|\"))</string>
189191
<key>patterns</key>
190192
<array>
191193
<dict>
@@ -287,7 +289,7 @@
287289
<key>begin</key>
288290
<string>(\'|\")</string>
289291
<key>end</key>
290-
<string>(\1)|((?&lt;!\\)\n)</string>
292+
<string>(\1)|(\n)</string>
291293
<key>beginCaptures</key>
292294
<dict>
293295
<key>1</key>
@@ -327,7 +329,7 @@
327329
<key>begin</key>
328330
<string>([rR])(\'|\")</string>
329331
<key>end</key>
330-
<string>(\2)|((?&lt;!\\)\n)</string>
332+
<string>(\2)|(\n)</string>
331333
<key>beginCaptures</key>
332334
<dict>
333335
<key>1</key>
@@ -7365,7 +7367,7 @@ indirectly through syntactic constructs
73657367
<key>name</key>
73667368
<string>string.quoted.single.python</string>
73677369
<key>begin</key>
7368-
<string>(\b[rR](?=[uU]))?([uU])?((['"]))</string>
7370+
<string>(?:\b([rR])(?=[uU]))?([uU])?((['"]))</string>
73697371
<key>end</key>
73707372
<string>(\3)|((?&lt;!\\)\n)</string>
73717373
<key>beginCaptures</key>
@@ -7665,7 +7667,7 @@ indirectly through syntactic constructs
76657667
<key>name</key>
76667668
<string>string.quoted.multi.python</string>
76677669
<key>begin</key>
7668-
<string>(\b[rR](?=[uU]))?([uU])?('''|""")</string>
7670+
<string>(?:\b([rR])(?=[uU]))?([uU])?('''|""")</string>
76697671
<key>end</key>
76707672
<string>(\3)</string>
76717673
<key>beginCaptures</key>

‎grammars/src/MagicPython.syntax.yaml

+7-3
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,11 @@ repository:
279279

280280
docstring-statement:
281281
begin: ^(?=\s*[rR]?(\'\'\'|\"\"\"|\'|\"))
282-
end: (?<=\'\'\'|\"\"\"|\'|\")
282+
comment: the string either terminates correctly or by the
283+
beginning of a new line (this is for single line
284+
docstrings that aren't terminated) AND it's not followed
285+
by another docstring
286+
end: ((?<=\1)|^)(?!\s*[rR]?(\'\'\'|\"\"\"|\'|\"))
283287
patterns:
284288
- include: '#docstring'
285289

@@ -312,7 +316,7 @@ repository:
312316

313317
- name: string.quoted.docstring.single.python
314318
begin: (\'|\")
315-
end: (\1)|((?<!\\)\n)
319+
end: (\1)|(\n)
316320
beginCaptures:
317321
'1': {name: punctuation.definition.string.begin.python}
318322
endCaptures:
@@ -324,7 +328,7 @@ repository:
324328

325329
- name: string.quoted.docstring.raw.single.python
326330
begin: ([rR])(\'|\")
327-
end: (\2)|((?<!\\)\n)
331+
end: (\2)|(\n)
328332
beginCaptures:
329333
'1': {name: storage.type.string.python}
330334
'2': {name: punctuation.definition.string.begin.python}

‎grammars/src/pystring.inc.syntax.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ repository:
4444

4545
string-quoted-${line}-line:
4646
name: string.quoted.${line}.python
47-
begin: (\b[rR](?=[uU]))?([uU])?(${marker})
47+
begin: (?:\b([rR])(?=[uU]))?([uU])?(${marker})
4848
end: (\3)${guard}
4949
beginCaptures:
5050
'1': {name: invalid.illegal.prefix.python}

‎package.json

+4
Original file line numberDiff line numberDiff line change
@@ -66,5 +66,9 @@
6666
"path": "./grammars/MagicPython.tmLanguage"
6767
}
6868
]
69+
},
70+
"dependencies": {
71+
"oniguruma": "^7.0.0",
72+
"syntaxdev": "0.0.16"
6973
}
7074
}

‎test/atom-spec/python-spec.js

+87-4
Original file line numberDiff line numberDiff line change
@@ -2856,7 +2856,7 @@ describe("Grammar Tests", function() {
28562856

28572857
it("test/docstrings/continuation2.py",
28582858
function() {
2859-
tokens = grammar.tokenizeLines("'\n'")
2859+
tokens = grammar.tokenizeLines("'\n'\n# comment")
28602860
expect(tokens[0][0].value).toBe("'");
28612861
expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
28622862
expect(tokens[0][1].value).toBe("");
@@ -2865,6 +2865,10 @@ describe("Grammar Tests", function() {
28652865
expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
28662866
expect(tokens[1][1].value).toBe("");
28672867
expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]);
2868+
expect(tokens[2][0].value).toBe("#");
2869+
expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
2870+
expect(tokens[2][1].value).toBe(" comment");
2871+
expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
28682872
});
28692873

28702874
it("test/docstrings/continuation3.py",
@@ -3035,6 +3039,73 @@ describe("Grammar Tests", function() {
30353039
expect(tokens[12][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]);
30363040
});
30373041

3042+
it("test/docstrings/continuation5.py",
3043+
function() {
3044+
tokens = grammar.tokenizeLines("'implicit ' \"concatenation\"\n\n'''implicit\n''' 'concatenation'\n\n'''implicit\n''' \"\"\"\nconcatenation\n\"\"\"\n\n'implicit' '''\nconcatenation\n'''")
3045+
expect(tokens[0][0].value).toBe("'");
3046+
expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3047+
expect(tokens[0][1].value).toBe("implicit ");
3048+
expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3049+
expect(tokens[0][2].value).toBe("'");
3050+
expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3051+
expect(tokens[0][3].value).toBe(" ");
3052+
expect(tokens[0][3].scopes).toEqual(["source.python"]);
3053+
expect(tokens[0][4].value).toBe("\"");
3054+
expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3055+
expect(tokens[0][5].value).toBe("concatenation");
3056+
expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3057+
expect(tokens[0][6].value).toBe("\"");
3058+
expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3059+
expect(tokens[1][0].value).toBe("");
3060+
expect(tokens[1][0].scopes).toEqual(["source.python"]);
3061+
expect(tokens[2][0].value).toBe("'''");
3062+
expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3063+
expect(tokens[2][1].value).toBe("implicit");
3064+
expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3065+
expect(tokens[3][0].value).toBe("'''");
3066+
expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3067+
expect(tokens[3][1].value).toBe(" ");
3068+
expect(tokens[3][1].scopes).toEqual(["source.python"]);
3069+
expect(tokens[3][2].value).toBe("'");
3070+
expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3071+
expect(tokens[3][3].value).toBe("concatenation");
3072+
expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3073+
expect(tokens[3][4].value).toBe("'");
3074+
expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3075+
expect(tokens[4][0].value).toBe("");
3076+
expect(tokens[4][0].scopes).toEqual(["source.python"]);
3077+
expect(tokens[5][0].value).toBe("'''");
3078+
expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3079+
expect(tokens[5][1].value).toBe("implicit");
3080+
expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3081+
expect(tokens[6][0].value).toBe("'''");
3082+
expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3083+
expect(tokens[6][1].value).toBe(" ");
3084+
expect(tokens[6][1].scopes).toEqual(["source.python"]);
3085+
expect(tokens[6][2].value).toBe("\"\"\"");
3086+
expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3087+
expect(tokens[7][0].value).toBe("concatenation");
3088+
expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3089+
expect(tokens[8][0].value).toBe("\"\"\"");
3090+
expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3091+
expect(tokens[9][0].value).toBe("");
3092+
expect(tokens[9][0].scopes).toEqual(["source.python"]);
3093+
expect(tokens[10][0].value).toBe("'");
3094+
expect(tokens[10][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3095+
expect(tokens[10][1].value).toBe("implicit");
3096+
expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3097+
expect(tokens[10][2].value).toBe("'");
3098+
expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3099+
expect(tokens[10][3].value).toBe(" ");
3100+
expect(tokens[10][3].scopes).toEqual(["source.python"]);
3101+
expect(tokens[10][4].value).toBe("'''");
3102+
expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3103+
expect(tokens[11][0].value).toBe("concatenation");
3104+
expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3105+
expect(tokens[12][0].value).toBe("'''");
3106+
expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3107+
});
3108+
30383109
it("test/docstrings/def1.py",
30393110
function() {
30403111
tokens = grammar.tokenizeLines("def foo():\n '''TE\\'''ST'''\n\ndef foo():\n r'''TE\\'''ST'''\n\ndef foo():\n R'''TE\\'''ST'''\n\ndef foo():\n u'''TEST'''\n\ndef foo():\n U'''TEST'''\n\ndef foo():\n b'''TEST'''\n\ndef foo():\n B'''TEST'''")
@@ -6498,7 +6569,7 @@ describe("Grammar Tests", function() {
64986569

64996570
it("test/fstrings/nested3.py",
65006571
function() {
6501-
tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"")
6572+
tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\n# comment")
65026573
expect(tokens[0][0].value).toBe("f");
65036574
expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]);
65046575
expect(tokens[0][1].value).toBe("\"");
@@ -6551,6 +6622,10 @@ describe("Grammar Tests", function() {
65516622
expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
65526623
expect(tokens[1][5].value).toBe("");
65536624
expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]);
6625+
expect(tokens[2][0].value).toBe("#");
6626+
expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
6627+
expect(tokens[2][1].value).toBe(" comment");
6628+
expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
65546629
});
65556630

65566631
it("test/fstrings/nested4.py",
@@ -9790,7 +9865,7 @@ describe("Grammar Tests", function() {
97909865

97919866
it("test/illegals/backticks3.py",
97929867
function() {
9793-
tokens = grammar.tokenizeLines("a = lambda `123`")
9868+
tokens = grammar.tokenizeLines("a = lambda `123`\n# comment")
97949869
expect(tokens[0][0].value).toBe("a");
97959870
expect(tokens[0][0].scopes).toEqual(["source.python"]);
97969871
expect(tokens[0][1].value).toBe(" ");
@@ -9811,6 +9886,10 @@ describe("Grammar Tests", function() {
98119886
expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]);
98129887
expect(tokens[0][9].value).toBe("");
98139888
expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python"]);
9889+
expect(tokens[1][0].value).toBe("#");
9890+
expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
9891+
expect(tokens[1][1].value).toBe(" comment");
9892+
expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
98149893
});
98159894

98169895
it("test/illegals/illegal1.py",
@@ -10951,7 +11030,7 @@ describe("Grammar Tests", function() {
1095111030

1095211031
it("test/regexp/python2.py",
1095311032
function() {
10954-
tokens = grammar.tokenizeLines("a = r'\n (?x)\n foo\n'")
11033+
tokens = grammar.tokenizeLines("a = r'\n (?x)\n foo\n'\n# comment")
1095511034
expect(tokens[0][0].value).toBe("a");
1095611035
expect(tokens[0][0].scopes).toEqual(["source.python"]);
1095711036
expect(tokens[0][1].value).toBe(" ");
@@ -10984,6 +11063,10 @@ describe("Grammar Tests", function() {
1098411063
expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
1098511064
expect(tokens[3][1].value).toBe("");
1098611065
expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]);
11066+
expect(tokens[4][0].value).toBe("#");
11067+
expect(tokens[4][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
11068+
expect(tokens[4][1].value).toBe(" comment");
11069+
expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
1098711070
});
1098811071

1098911072
it("test/regexp/python3.py",

‎test/docstrings/continuation2.py

+3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
'
22
'
3+
# comment
34

45

56

67
' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
78
: invalid.illegal.newline.python, source.python, string.quoted.docstring.single.python
89
' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
910
: invalid.illegal.newline.python, source.python, string.quoted.docstring.single.python
11+
# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
12+
comment : comment.line.number-sign.python, source.python

‎test/docstrings/continuation5.py

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
'implicit ' "concatenation"
2+
3+
'''implicit
4+
''' 'concatenation'
5+
6+
'''implicit
7+
''' """
8+
concatenation
9+
"""
10+
11+
'implicit' '''
12+
concatenation
13+
'''
14+
15+
16+
17+
' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
18+
implicit : source.python, string.quoted.docstring.single.python
19+
' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python
20+
: source.python
21+
" : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
22+
concatenation : source.python, string.quoted.docstring.single.python
23+
" : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python
24+
: source.python
25+
''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
26+
implicit : source.python, string.quoted.docstring.multi.python
27+
''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
28+
: source.python
29+
' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
30+
concatenation : source.python, string.quoted.docstring.single.python
31+
' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python
32+
: source.python
33+
''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
34+
implicit : source.python, string.quoted.docstring.multi.python
35+
''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
36+
: source.python
37+
""" : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
38+
concatenation : source.python, string.quoted.docstring.multi.python
39+
""" : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python
40+
: source.python
41+
' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python
42+
implicit : source.python, string.quoted.docstring.single.python
43+
' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python
44+
: source.python
45+
''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python
46+
concatenation : source.python, string.quoted.docstring.multi.python
47+
''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python

‎test/fstrings/nested3.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
f"result: {value:{60}.{16!s:2}{'qwerty'
22
[2]}}"
3-
3+
# comment
44

55

66

@@ -30,3 +30,5 @@
3030
}} : source.python
3131
" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
3232
: invalid.illegal.newline.python, source.python, string.quoted.single.python
33+
# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
34+
comment : comment.line.number-sign.python, source.python

0 commit comments

Comments
 (0)