Skip to content

Commit 51224e7

Browse files
author
Mikhail Arkhipov
authored
Format on type fixes (#1798)
* Undo changes * Test fixes * Increase timeout * Remove double event listening * Remove test * Revert "Remove test" This reverts commit e240c3f. * Revert "Remove double event listening" This reverts commit af573be. * #1096 The if statement is automatically formatted incorrectly * Merge fix * Add more tests * More tests * Typo * Test * Also better handle multiline arguments * Add a couple missing periods [skip ci] * Undo changes * Test fixes * Increase timeout * Remove double event listening * Remove test * Revert "Remove test" This reverts commit e240c3f. * Revert "Remove double event listening" This reverts commit af573be. * Merge fix * #1257 On type formatting errors for args and kwargs * Handle f-strings * Stop importing from test code * #1308 Single line statements leading to an indentation on the next line * #726 editing python after inline if statement invalid indent * Undo change * Move constant * Harden LS startup error checks * #1364 Intellisense doesn't work after specific const string * Telemetry for the analysis enging * PR feedback * Fix typo * Test baseline update * Jedi 0.12 * Priority to goto_defition * News * Replace unzip * Linux flavors + test * Grammar check * Grammar test * Test baselines * Add news * Pin dependency [skip ci] * Specify markdown as preferable format * Improve function argument detection * Specify markdown * Pythia setting * Baseline updates * Baseline update * Improve startup * Handle missing interpreter better * Handle interpreter change * Delete old file * Fix LS startup time reporting * Remove Async suffix from IFileSystem * Remove Pythia * Remove pre-packaged MSIL * Exe name on Unix * Plain linux * Fix casing * Fix message * Update PTVS engine activation steps * Type formatter eats space in from . * fIX CASING * Remove flag * Don't wait for LS * Small test fixes * Update hover baselines * Rename the engine * Formatting 1 * Add support for 'rf' strings * Add two spaces before comment per PEP * Fix @ operator spacing * Handle module and unary ops * Type hints * Fix typo * Trailing comma * Require space after if * Update list of keywords * PR feedback
1 parent 6d2d390 commit 51224e7

File tree

6 files changed

+160
-42
lines changed

6 files changed

+160
-42
lines changed

‎src/client/formatters/lineFormatter.ts

+64-19
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,21 @@ import { TextRangeCollection } from '../language/textRangeCollection';
1010
import { Tokenizer } from '../language/tokenizer';
1111
import { ITextRangeCollection, IToken, TokenType } from '../language/types';
1212

13+
const keywordsWithSpaceBeforeBrace = [
14+
'and', 'as', 'assert', 'await',
15+
'del',
16+
'except', 'elif',
17+
'for', 'from',
18+
'global',
19+
'if', 'import', 'in', 'is',
20+
'lambda',
21+
'nonlocal', 'not',
22+
'or',
23+
'raise', 'return',
24+
'while', 'with',
25+
'yield'
26+
];
27+
1328
export class LineFormatter {
1429
private builder = new TextBuilder();
1530
private tokens: ITextRangeCollection<IToken> = new TextRangeCollection<IToken>([]);
@@ -59,7 +74,7 @@ export class LineFormatter {
5974
}
6075
const id = this.text.substring(t.start, t.end);
6176
this.builder.append(id);
62-
if (this.keywordWithSpaceAfter(id) && next && this.isOpenBraceType(next.type)) {
77+
if (this.isKeywordWithSpaceBeforeBrace(id) && next && this.isOpenBraceType(next.type)) {
6378
// for x in ()
6479
this.builder.softAppendSpace();
6580
}
@@ -75,9 +90,9 @@ export class LineFormatter {
7590
break;
7691

7792
case TokenType.Comment:
78-
// Add space before in-line comment.
93+
// Add 2 spaces before in-line comment per PEP guidelines.
7994
if (prev) {
80-
this.builder.softAppendSpace();
95+
this.builder.softAppendSpace(2);
8196
}
8297
this.builder.append(this.text.substring(t.start, t.end));
8398
break;
@@ -98,28 +113,35 @@ export class LineFormatter {
98113
private handleOperator(index: number): void {
99114
const t = this.tokens.getItemAt(index);
100115
const prev = index > 0 ? this.tokens.getItemAt(index - 1) : undefined;
116+
const opCode = this.text.charCodeAt(t.start);
101117
const next = index < this.tokens.count - 1 ? this.tokens.getItemAt(index + 1) : undefined;
102118

103119
if (t.length === 1) {
104-
const opCode = this.text.charCodeAt(t.start);
105120
switch (opCode) {
106121
case Char.Equal:
107-
if (this.handleEqual(t, index)) {
108-
return;
109-
}
110-
break;
122+
this.handleEqual(t, index);
123+
return;
111124
case Char.Period:
112125
if (prev && this.isKeyword(prev, 'from')) {
113126
this.builder.softAppendSpace();
114127
}
115-
this.builder.append(this.text[t.start]);
128+
this.builder.append('.');
116129
if (next && this.isKeyword(next, 'import')) {
117130
this.builder.softAppendSpace();
118131
}
119132
return;
120133
case Char.At:
134+
if (prev) {
135+
// Binary case
136+
this.builder.softAppendSpace();
137+
this.builder.append('@');
138+
this.builder.softAppendSpace();
139+
} else {
140+
this.builder.append('@');
141+
}
142+
return;
121143
case Char.ExclamationMark:
122-
this.builder.append(this.text[t.start]);
144+
this.builder.append('!');
123145
return;
124146
case Char.Asterisk:
125147
if (prev && this.isKeyword(prev, 'lambda')) {
@@ -153,19 +175,34 @@ export class LineFormatter {
153175

154176
this.builder.softAppendSpace();
155177
this.builder.append(this.text.substring(t.start, t.end));
178+
179+
// Check unary case
180+
if (prev && prev.type === TokenType.Operator) {
181+
if (opCode === Char.Hyphen || opCode === Char.Plus || opCode === Char.Tilde) {
182+
return;
183+
}
184+
}
156185
this.builder.softAppendSpace();
157186
}
158187

159-
private handleEqual(t: IToken, index: number): boolean {
188+
private handleEqual(t: IToken, index: number): void {
160189
if (this.isMultipleStatements(index) && !this.braceCounter.isOpened(TokenType.OpenBrace)) {
161-
return false; // x = 1; x, y = y, x
190+
// x = 1; x, y = y, x
191+
this.builder.softAppendSpace();
192+
this.builder.append('=');
193+
this.builder.softAppendSpace();
194+
return;
162195
}
196+
163197
// Check if this is = in function arguments. If so, do not add spaces around it.
164198
if (this.isEqualsInsideArguments(index)) {
165199
this.builder.append('=');
166-
return true;
200+
return;
167201
}
168-
return false;
202+
203+
this.builder.softAppendSpace();
204+
this.builder.append('=');
205+
this.builder.softAppendSpace();
169206
}
170207

171208
private handleOther(t: IToken, index: number): void {
@@ -188,6 +225,12 @@ export class LineFormatter {
188225
return;
189226
}
190227

228+
if (t.type === TokenType.Number && prev && prev.type === TokenType.Operator && prev.length === 1 && this.text.charCodeAt(prev.start) === Char.Tilde) {
229+
// Special case for ~ before numbers
230+
this.builder.append(this.text.substring(t.start, t.end));
231+
return;
232+
}
233+
191234
if (t.type === TokenType.Unknown) {
192235
this.handleUnknown(t);
193236
} else {
@@ -224,6 +267,10 @@ export class LineFormatter {
224267
return false;
225268
}
226269

270+
if (index > 1 && this.tokens.getItemAt(index - 2).type === TokenType.Colon) {
271+
return false; // Type hint should have spaces around like foo(x: int = 1) per PEP 8
272+
}
273+
227274
const first = this.tokens.getItemAt(0);
228275
if (first.type === TokenType.Comma) {
229276
return true; // Line starts with commma
@@ -278,11 +325,9 @@ export class LineFormatter {
278325
}
279326
return false;
280327
}
281-
private keywordWithSpaceAfter(s: string): boolean {
282-
return s === 'in' || s === 'return' || s === 'and' ||
283-
s === 'or' || s === 'not' || s === 'from' ||
284-
s === 'import' || s === 'except' || s === 'for' ||
285-
s === 'as' || s === 'is';
328+
329+
private isKeywordWithSpaceBeforeBrace(s: string): boolean {
330+
return keywordsWithSpaceBeforeBrace.indexOf(s) >= 0;
286331
}
287332
private isKeyword(t: IToken, keyword: string): boolean {
288333
return t.type === TokenType.Identifier && t.length === keyword.length && this.text.substr(t.start, t.length) === keyword;

‎src/client/language/textBuilder.ts

+8-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,14 @@ export class TextBuilder {
1616
return this.segments.join('');
1717
}
1818

19-
public softAppendSpace(): void {
20-
if (!this.isLastWhiteSpace() && this.segments.length > 0) {
19+
public softAppendSpace(count: number = 1): void {
20+
if (this.segments.length === 0) {
21+
return;
22+
}
23+
if (this.isLastWhiteSpace()) {
24+
count = count - 1;
25+
}
26+
for (let i = 0; i < count; i += 1) {
2127
this.segments.push(' ');
2228
}
2329
}

‎src/client/language/tokenizer.ts

+31-12
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,8 @@ export class Tokenizer implements ITokenizer {
280280
case Char.Caret:
281281
case Char.Equal:
282282
case Char.ExclamationMark:
283+
case Char.Percent:
284+
case Char.Tilde:
283285
length = nextChar === Char.Equal ? 2 : 1;
284286
break;
285287

@@ -350,23 +352,40 @@ export class Tokenizer implements ITokenizer {
350352
this.tokens.push(new Token(TokenType.Comment, start, this.cs.position - start));
351353
}
352354

355+
// tslint:disable-next-line:cyclomatic-complexity
353356
private getStringPrefixLength(): number {
354-
if (this.cs.currentChar === Char.f && (this.cs.nextChar === Char.SingleQuote || this.cs.nextChar === Char.DoubleQuote)) {
355-
return 1; // f-string
357+
if (this.cs.currentChar === Char.SingleQuote || this.cs.currentChar === Char.DoubleQuote) {
358+
return 0; // Simple string, no prefix
356359
}
357-
if (this.cs.currentChar === Char.b || this.cs.currentChar === Char.B || this.cs.currentChar === Char.u || this.cs.currentChar === Char.U) {
358-
if (this.cs.nextChar === Char.SingleQuote || this.cs.nextChar === Char.DoubleQuote) {
359-
// b-string or u-string
360-
return 1;
360+
361+
if (this.cs.nextChar === Char.SingleQuote || this.cs.nextChar === Char.DoubleQuote) {
362+
switch (this.cs.currentChar) {
363+
case Char.f:
364+
case Char.F:
365+
case Char.r:
366+
case Char.R:
367+
case Char.b:
368+
case Char.B:
369+
case Char.u:
370+
case Char.U:
371+
return 1; // single-char prefix like u"" or r""
372+
default:
373+
break;
361374
}
362-
if (this.cs.nextChar === Char.r || this.cs.nextChar === Char.R) {
363-
// b-string or u-string with 'r' suffix
364-
if (this.cs.lookAhead(2) === Char.SingleQuote || this.cs.lookAhead(2) === Char.DoubleQuote) {
365-
return 2;
366-
}
375+
}
376+
377+
if (this.cs.lookAhead(2) === Char.SingleQuote || this.cs.lookAhead(2) === Char.DoubleQuote) {
378+
const prefix = this.cs.getText().substr(this.cs.position, 2).toLowerCase();
379+
switch (prefix) {
380+
case 'rf':
381+
case 'ur':
382+
case 'br':
383+
return 2;
384+
default:
385+
break;
367386
}
368387
}
369-
return this.cs.currentChar === Char.SingleQuote || this.cs.currentChar === Char.DoubleQuote ? 0 : -1;
388+
return -1;
370389
}
371390

372391
private getQuoteType(): QuoteType {

‎src/test/format/extension.lineFormatter.test.ts

+32-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ suite('Formatting - line formatter', () => {
5959
testFormatLine('[ 1 :[2: (x,),y]]{1}', '[1:[2:(x,), y]]{1}');
6060
});
6161
test('Trailing comment', () => {
62-
testFormatLine('x=1 # comment', 'x = 1 # comment');
62+
testFormatLine('x=1 # comment', 'x = 1 # comment');
6363
});
6464
test('Single comment', () => {
6565
testFormatLine('# comment', '# comment');
@@ -87,9 +87,14 @@ suite('Formatting - line formatter', () => {
8787
});
8888
test('Brace after keyword', () => {
8989
testFormatLine('for x in(1,2,3)', 'for x in (1, 2, 3)');
90+
testFormatLine('assert(1,2,3)', 'assert (1, 2, 3)');
91+
testFormatLine('if (True|False)and(False/True)not (! x )', 'if (True | False) and (False / True) not (!x)');
92+
testFormatLine('while (True|False)', 'while (True | False)');
93+
testFormatLine('yield(a%b)', 'yield (a % b)');
9094
});
9195
test('Dot operator', () => {
9296
testFormatLine('x.y', 'x.y');
97+
testFormatLine('5 .y', '5.y');
9398
});
9499
test('Unknown tokens no space', () => {
95100
testFormatLine('abc\\n\\', 'abc\\n\\');
@@ -121,6 +126,32 @@ suite('Formatting - line formatter', () => {
121126
test('from..x import', () => {
122127
testFormatLine('from..x import', 'from ..x import');
123128
});
129+
test('Raw strings', () => {
130+
testFormatLine('z=r""', 'z = r""');
131+
testFormatLine('z=rf""', 'z = rf""');
132+
testFormatLine('z=R""', 'z = R""');
133+
testFormatLine('z=RF""', 'z = RF""');
134+
});
135+
test('Binary @', () => {
136+
testFormatLine('a@ b', 'a @ b');
137+
});
138+
test('Unary operators', () => {
139+
testFormatLine('x= - y', 'x = -y');
140+
testFormatLine('x= + y', 'x = +y');
141+
testFormatLine('x= ~ y', 'x = ~y');
142+
testFormatLine('x=-1', 'x = -1');
143+
testFormatLine('x= +1', 'x = +1');
144+
testFormatLine('x= ~1 ', 'x = ~1');
145+
});
146+
test('Equals with type hints', () => {
147+
testFormatLine('def foo(x:int=3,x=100.)', 'def foo(x: int = 3, x=100.)');
148+
});
149+
test('Trailing comma', () => {
150+
testFormatLine('a, =[1]', 'a, = [1]');
151+
});
152+
test('if()', () => {
153+
testFormatLine('if(True) :', 'if (True):');
154+
});
124155
test('Grammar file', () => {
125156
const content = fs.readFileSync(grammarFile).toString('utf8');
126157
const lines = content.splitLines({ trim: false, removeEmptyEntries: false });

‎src/test/language/tokenizer.test.ts

+22-5
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ suite('Language.Tokenizer', () => {
185185
});
186186
test('Unknown token', () => {
187187
const t = new Tokenizer();
188-
const tokens = t.tokenize('~$');
188+
const tokens = t.tokenize('`$');
189189
assert.equal(tokens.count, 1);
190190

191191
assert.equal(tokens.getItemAt(0).type, TokenType.Unknown);
@@ -301,20 +301,37 @@ suite('Language.Tokenizer', () => {
301301
assert.equal(tokens.getItemAt(5).type, TokenType.Number);
302302
assert.equal(tokens.getItemAt(5).length, 5);
303303
});
304+
test('Simple expression, leading minus', () => {
305+
const t = new Tokenizer();
306+
const tokens = t.tokenize('x == -y');
307+
assert.equal(tokens.count, 4);
308+
309+
assert.equal(tokens.getItemAt(0).type, TokenType.Identifier);
310+
assert.equal(tokens.getItemAt(0).length, 1);
311+
312+
assert.equal(tokens.getItemAt(1).type, TokenType.Operator);
313+
assert.equal(tokens.getItemAt(1).length, 2);
314+
315+
assert.equal(tokens.getItemAt(2).type, TokenType.Operator);
316+
assert.equal(tokens.getItemAt(2).length, 1);
317+
318+
assert.equal(tokens.getItemAt(3).type, TokenType.Identifier);
319+
assert.equal(tokens.getItemAt(3).length, 1);
320+
});
304321
test('Operators', () => {
305322
const text = '< <> << <<= ' +
306323
'== != > >> >>= >= <=' +
307-
'+ -' +
324+
'+ - ~ %' +
308325
'* ** / /= //=' +
309-
'*= += -= **= ' +
326+
'*= += -= ~= %= **= ' +
310327
'& &= | |= ^ ^= ->';
311328
const tokens = new Tokenizer().tokenize(text);
312329
const lengths = [
313330
1, 2, 2, 3,
314331
2, 2, 1, 2, 3, 2, 2,
315-
1, 1,
332+
1, 1, 1, 1,
316333
1, 2, 1, 2, 3,
317-
2, 2, 2, 3,
334+
2, 2, 2, 2, 2, 3,
318335
1, 2, 1, 2, 1, 2, 2];
319336
assert.equal(tokens.count, lengths.length);
320337
for (let i = 0; i < tokens.count; i += 1) {

‎src/test/pythonFiles/formatting/pythonGrammar.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ def test_eof_error(self):
236236
compile(s, "<test>", "exec")
237237
self.assertIn("unexpected EOF", str(cm.exception))
238238

239-
var_annot_global: int # a global annotated is necessary for test_var_annot
239+
var_annot_global: int # a global annotated is necessary for test_var_annot
240240

241241
# custom namespace for testing __annotations__
242242

@@ -643,7 +643,7 @@ def test_lambdef(self):
643643
### lambdef: 'lambda' [varargslist] ':' test
644644
l1 = lambda: 0
645645
self.assertEqual(l1(), 0)
646-
l2 = lambda: a[d] # XXX just testing the expression
646+
l2 = lambda: a[d] # XXX just testing the expression
647647
l3 = lambda: [2 < x for x in [-1, 3, 0]]
648648
self.assertEqual(l3(), [0, 1, 0])
649649
l4 = lambda x=lambda y=lambda z=1: z: y(): x()
@@ -1492,7 +1492,7 @@ def __imatmul__(self, o):
14921492
self.other = o
14931493
return self
14941494
m = M()
1495-
self.assertEqual(m@m, 4)
1495+
self.assertEqual(m @ m, 4)
14961496
m @= 42
14971497
self.assertEqual(m.other, 42)
14981498

0 commit comments

Comments
 (0)