Fix handling of escaped string in tokenizer + PTVS launch hardening (#1377)

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* #1096 The if statement is automatically formatted incorrectly

* Merge fix

* Add more tests

* More tests

* Typo

* Test

* Also better handle multiline arguments

* Add a couple missing periods

[skip ci]

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Merge fix

* #1257 On type formatting errors for args and kwargs

* Handle f-strings

* Stop importing from test code

* #1308 Single line statements leading to an indentation on the next line

* #726 editing python after inline if statement invalid indent

* Undo change

* Move constant

* Harden LS startup error checks

* #1364 Intellisense doesn't work after specific const string

Author: Mikhail Arkhipov

src/client/activation/analysis.ts

@@ -3,9 +3,11 @@
 
 import * as path from 'path';
 import { ExtensionContext, OutputChannel } from 'vscode';
-import { Disposable, LanguageClient, LanguageClientOptions, ServerOptions } from 'vscode-languageclient';
+import { Message } from 'vscode-jsonrpc';
+import { CloseAction, Disposable, ErrorAction, ErrorHandler, LanguageClient, LanguageClientOptions, ServerOptions } from 'vscode-languageclient';
 import { IApplicationShell } from '../common/application/types';
 import { isTestExecution, STANDARD_OUTPUT_CHANNEL } from '../common/constants';
+import { createDeferred, Deferred } from '../common/helpers';
 import { IFileSystem, IPlatformService } from '../common/platform/types';
 import { IProcessService } from '../common/process/types';
 import { StopWatch } from '../common/stopWatch';
@@ -21,6 +23,18 @@ const dotNetCommand = 'dotnet';
 const languageClientName = 'Python Tools';
 const analysisEngineFolder = 'analysis';
 
+class LanguageServerStartupErrorHandler implements ErrorHandler {
+    constructor(private readonly deferred: Deferred<void>) { }
+    public error(error: Error, message: Message, count: number): ErrorAction {
+        this.deferred.reject();
+        return ErrorAction.Shutdown;
+    }
+    public closed(): CloseAction {
+        this.deferred.reject();
+        return CloseAction.DoNotRestart;
+    }
+}
+
 export class AnalysisExtensionActivator implements IExtensionActivator {
     private readonly configuration: IConfigurationService;
     private readonly appShell: IApplicationShell;
@@ -92,16 +106,23 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
 
     private async tryStartLanguageClient(context: ExtensionContext, lc: LanguageClient): Promise<void> {
         let disposable: Disposable | undefined;
+        const deferred = createDeferred<void>();
         try {
+            lc.clientOptions.errorHandler = new LanguageServerStartupErrorHandler(deferred);
+
             disposable = lc.start();
-            await lc.onReady();
+            lc.onReady()
+                .then(() => deferred.resolve())
+                .catch(ex => deferred.reject());
+            await deferred.promise;
+
             this.output.appendLine(`Language server ready: ${this.sw.elapsedTime} ms`);
             context.subscriptions.push(disposable);
         } catch (ex) {
             if (disposable) {
                 disposable.dispose();
-                throw ex;
             }
+            throw ex;
         }
     }
 
@@ -157,12 +178,8 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
         // tslint:disable-next-line:no-string-literal
         properties['SearchPaths'] = searchPaths;
 
-        if (isTestExecution()) {
-            // tslint:disable-next-line:no-string-literal
-            properties['TestEnvironment'] = true;
-        }
-
         const selector: string[] = [PYTHON];
+
         // Options to control the language client
         return {
             // Register the server for Python documents
@@ -181,7 +198,8 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
                     trimDocumentationText: false,
                     maxDocumentationTextLength: 0
                 },
-                asyncStartup: true
+                asyncStartup: true,
+                testEnvironment: isTestExecution()
             }
         };
     }

src/client/language/tokenizer.ts

@@ -366,6 +366,9 @@ export class Tokenizer implements ITokenizer {
 
     private skipToSingleEndQuote(quote: number): void {
         while (!this.cs.isEndOfStream()) {
+            if (this.cs.currentChar === Char.LineFeed || this.cs.currentChar === Char.CarriageReturn) {
+                return; // Unterminated single-line string
+            }
             if (this.cs.currentChar === Char.Backslash && this.cs.nextChar === quote) {
                 this.cs.advance(2);
                 continue;

src/test/language/tokenizer.test.ts

@@ -116,6 +116,23 @@ suite('Language.Tokenizer', () => {
         assert.equal(tokens.getItemAt(0).type, TokenType.String);
         assert.equal(tokens.getItemAt(0).length, 14);
     });
+    test('Strings: escape at the end of single quoted string ', async () => {
+        const t = new Tokenizer();
+        // tslint:disable-next-line:quotemark
+        const tokens = t.tokenize("'quoted\\'\nx");
+        assert.equal(tokens.count, 2);
+        assert.equal(tokens.getItemAt(0).type, TokenType.String);
+        assert.equal(tokens.getItemAt(0).length, 9);
+        assert.equal(tokens.getItemAt(1).type, TokenType.Identifier);
+    });
+    test('Strings: escape at the end of double quoted string ', async () => {
+        const t = new Tokenizer();
+        const tokens = t.tokenize('"quoted\\"\nx');
+        assert.equal(tokens.count, 2);
+        assert.equal(tokens.getItemAt(0).type, TokenType.String);
+        assert.equal(tokens.getItemAt(0).length, 9);
+        assert.equal(tokens.getItemAt(1).type, TokenType.Identifier);
+    });
     test('Comments', async () => {
         const t = new Tokenizer();
         const tokens = t.tokenize(' #co"""mment1\n\t\n#comm\'ent2 ');