PTVS engine update + handling of the interpreter change (microsoft#1613)

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* microsoft#1096 The if statement is automatically formatted incorrectly

* Merge fix

* Add more tests

* More tests

* Typo

* Test

* Also better handle multiline arguments

* Add a couple missing periods

[skip ci]

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Merge fix

* microsoft#1257 On type formatting errors for args and kwargs

* Handle f-strings

* Stop importing from test code

* microsoft#1308 Single line statements leading to an indentation on the next line

* microsoft#726 editing python after inline if statement invalid indent

* Undo change

* Move constant

* Harden LS startup error checks

* microsoft#1364 Intellisense doesn't work after specific const string

* Telemetry for the analysis enging

* PR feedback

* Fix typo

* Test baseline update

* Jedi 0.12

* Priority to goto_defition

* News

* Replace unzip

* Linux flavors + test

* Grammar check

* Grammar test

* Test baselines

* Add news

* Pin dependency

[skip ci]

* Specify markdown as preferable format

* Improve function argument detection

* Specify markdown

* Pythia setting

* Baseline updates

* Baseline update

* Improve startup

* Handle missing interpreter better

* Handle interpreter change

* Delete old file

* Fix LS startup time reporting

* Remove Async suffix from IFileSystem

Author: Mikhail Arkhipov

package.json

@@ -1232,6 +1232,18 @@
                     "description": "Whether to install Python modules globally when not using an environment.",
                     "scope": "resource"
                 },
+                "python.pythiaEnabled": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Enables AI-driven additions to the completion list. Does not apply to Jedi.",
+                    "scope": "resource"
+                },
+                "python.jediEnabled": {
+                    "type": "boolean",
+                    "default": true,
+                    "description": "Enables Jedi as IntelliSense engine instead of Microsoft Python Analysis Engine.",
+                    "scope": "resource"
+                },
                 "python.jediMemoryLimit": {
                     "type": "number",
                     "default": 0,

src/client/activation/analysis.ts

@@ -13,6 +13,7 @@ import { IProcessServiceFactory } from '../common/process/types';
 import { StopWatch } from '../common/stopWatch';
 import { IConfigurationService, IOutputChannel, IPythonSettings } from '../common/types';
 import { IEnvironmentVariablesProvider } from '../common/variables/types';
+import { IInterpreterService } from '../interpreter/contracts';
 import { IServiceContainer } from '../ioc/types';
 import {
     PYTHON_ANALYSIS_ENGINE_DOWNLOADED,
@@ -35,11 +36,11 @@ class LanguageServerStartupErrorHandler implements ErrorHandler {
     constructor(private readonly deferred: Deferred<void>) { }
     public error(error: Error, message: Message, count: number): ErrorAction {
         this.deferred.reject(error);
-        return ErrorAction.Shutdown;
+        return ErrorAction.Continue;
     }
     public closed(): CloseAction {
         this.deferred.reject();
-        return CloseAction.DoNotRestart;
+        return CloseAction.Restart;
     }
 }
 
@@ -50,39 +51,66 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
     private readonly fs: IFileSystem;
     private readonly sw = new StopWatch();
     private readonly platformData: PlatformData;
+    private readonly interpreterService: IInterpreterService;
+    private readonly disposables: Disposable[] = [];
     private languageClient: LanguageClient | undefined;
+    private context: ExtensionContext | undefined;
+    private interpreterHash: string = '';
 
     constructor(private readonly services: IServiceContainer, pythonSettings: IPythonSettings) {
         this.configuration = this.services.get<IConfigurationService>(IConfigurationService);
         this.appShell = this.services.get<IApplicationShell>(IApplicationShell);
         this.output = this.services.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
         this.fs = this.services.get<IFileSystem>(IFileSystem);
         this.platformData = new PlatformData(services.get<IPlatformService>(IPlatformService), this.fs);
+        this.interpreterService = this.services.get<IInterpreterService>(IInterpreterService);
     }
 
     public async activate(context: ExtensionContext): Promise<boolean> {
+        this.sw.reset();
+        this.context = context;
         const clientOptions = await this.getAnalysisOptions(context);
         if (!clientOptions) {
             return false;
         }
+        this.disposables.push(this.interpreterService.onDidChangeInterpreter(() => this.restartLanguageServer()));
         return this.startLanguageServer(context, clientOptions);
     }
 
     public async deactivate(): Promise<void> {
         if (this.languageClient) {
             await this.languageClient.stop();
         }
+        for (const d of this.disposables) {
+            d.dispose();
+        }
+    }
+
+    private async restartLanguageServer(): Promise<void> {
+        if (!this.context) {
+            return;
+        }
+        const ids = new InterpreterDataService(this.context, this.services);
+        const idata = await ids.getInterpreterData();
+        if (!idata || idata.hash !== this.interpreterHash) {
+            this.interpreterHash = idata ? idata.hash : '';
+            await this.deactivate();
+            await this.activate(this.context);
+        }
     }
 
     private async startLanguageServer(context: ExtensionContext, clientOptions: LanguageClientOptions): Promise<boolean> {
         // Determine if we are running MSIL/Universal via dotnet or self-contained app.
         const mscorlib = path.join(context.extensionPath, analysisEngineFolder, 'mscorlib.dll');
+        const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
         let downloadPackage = false;
 
         const reporter = getTelemetryReporter();
         reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ENABLED);
 
-        if (!await this.fs.fileExistsAsync(mscorlib)) {
+        await this.checkPythiaModel(context, downloader);
+
+        if (!await this.fs.fileExists(mscorlib)) {
             // Depends on .NET Runtime or SDK
             this.languageClient = this.createSimpleLanguageClient(context, clientOptions);
             try {
@@ -100,7 +128,7 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
         }
 
         if (downloadPackage) {
-            const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
+            this.appShell.showWarningMessage('.NET Runtime is not found, platform-specific Python Analysis Engine will be downloaded.');
             await downloader.downloadAnalysisEngine(context);
             reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_DOWNLOADED);
         }
@@ -128,7 +156,9 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
             disposable = lc.start();
             lc.onReady()
                 .then(() => deferred.resolve())
-                .catch(deferred.reject);
+                .catch((reason) => {
+                    deferred.reject(reason);
+                });
             await deferred.promise;
 
             this.output.appendLine(`Language server ready: ${this.sw.elapsedTime} ms`);
@@ -172,20 +202,19 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
         const interpreterData = await interpreterDataService.getInterpreterData();
         if (!interpreterData) {
             const appShell = this.services.get<IApplicationShell>(IApplicationShell);
-            appShell.showErrorMessage('Unable to determine path to Python interpreter.');
-            return;
+            appShell.showWarningMessage('Unable to determine path to Python interpreter. IntelliSense will be limited.');
         }
 
-        // tslint:disable-next-line:no-string-literal
-        properties['InterpreterPath'] = interpreterData.path;
-        // tslint:disable-next-line:no-string-literal
-        properties['Version'] = interpreterData.version;
-        // tslint:disable-next-line:no-string-literal
-        properties['PrefixPath'] = interpreterData.prefix;
-        // tslint:disable-next-line:no-string-literal
-        properties['DatabasePath'] = path.join(context.extensionPath, analysisEngineFolder);
+        if (interpreterData) {
+            // tslint:disable-next-line:no-string-literal
+            properties['InterpreterPath'] = interpreterData.path;
+            // tslint:disable-next-line:no-string-literal
+            properties['Version'] = interpreterData.version;
+            // tslint:disable-next-line:no-string-literal
+            properties['PrefixPath'] = interpreterData.prefix;
+        }
 
-        let searchPaths = interpreterData.searchPaths;
+        let searchPaths = interpreterData ? interpreterData.searchPaths : '';
         const settings = this.configuration.getSettings();
         if (settings.autoComplete) {
             const extraPaths = settings.autoComplete.extraPaths;
@@ -194,12 +223,15 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
             }
         }
 
+        // tslint:disable-next-line:no-string-literal
+        properties['DatabasePath'] = path.join(context.extensionPath, analysisEngineFolder);
+
         const envProvider = this.services.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
         const pythonPath = (await envProvider.getEnvironmentVariables()).PYTHONPATH;
+        this.interpreterHash = interpreterData ? interpreterData.hash : '';
 
         // tslint:disable-next-line:no-string-literal
         properties['SearchPaths'] = `${searchPaths};${pythonPath ? pythonPath : ''}`;
-
         const selector: string[] = [PYTHON];
 
         // Options to control the language client
@@ -215,12 +247,14 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
                     properties
                 },
                 displayOptions: {
+                    preferredFormat: 1, // Markdown
                     trimDocumentationLines: false,
                     maxDocumentationLineLength: 0,
                     trimDocumentationText: false,
                     maxDocumentationTextLength: 0
                 },
                 asyncStartup: true,
+                pythiaEnabled: settings.pythiaEnabled,
                 testEnvironment: isTestExecution()
             }
         };
@@ -231,4 +265,11 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
         const result = await ps.exec('dotnet', ['--version']).catch(() => { return { stdout: '' }; });
         return result.stdout.trim().startsWith('2.');
     }
+
+    private async checkPythiaModel(context: ExtensionContext, downloader: AnalysisEngineDownloader): Promise<void> {
+        const settings = this.configuration.getSettings();
+        if (settings.pythiaEnabled) {
+            await downloader.downloadPythiaModel(context);
+        }
+    }
 }

src/client/activation/downloader.ts

@@ -1,13 +1,12 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import * as fs from 'fs';
+import * as fileSystem from 'fs';
 import * as path from 'path';
 import * as request from 'request';
 import * as requestProgress from 'request-progress';
 import { ExtensionContext, OutputChannel, ProgressLocation, window } from 'vscode';
 import { STANDARD_OUTPUT_CHANNEL } from '../common/constants';
-import { noop } from '../common/core.utils';
 import { createDeferred, createTemporaryFile } from '../common/helpers';
 import { IFileSystem, IPlatformService } from '../common/platform/types';
 import { IOutputChannel } from '../common/types';
@@ -22,49 +21,78 @@ const downloadUriPrefix = 'https://pvsc.blob.core.windows.net/python-analysis';
 const downloadBaseFileName = 'python-analysis-vscode';
 const downloadVersion = '0.1.0';
 const downloadFileExtension = '.nupkg';
+const pythiaModelName = 'model-sequence.json.gz';
 
 export class AnalysisEngineDownloader {
     private readonly output: OutputChannel;
     private readonly platform: IPlatformService;
     private readonly platformData: PlatformData;
+    private readonly fs: IFileSystem;
 
     constructor(private readonly services: IServiceContainer, private engineFolder: string) {
         this.output = this.services.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
+        this.fs = this.services.get<IFileSystem>(IFileSystem);
         this.platform = this.services.get<IPlatformService>(IPlatformService);
-        this.platformData = new PlatformData(this.platform, this.services.get<IFileSystem>(IFileSystem));
+        this.platformData = new PlatformData(this.platform, this.fs);
     }
 
     public async downloadAnalysisEngine(context: ExtensionContext): Promise<void> {
-        const localTempFilePath = await this.downloadFile();
+        const platformString = await this.platformData.getPlatformName();
+        const enginePackageFileName = `${downloadBaseFileName}-${platformString}.${downloadVersion}${downloadFileExtension}`;
+
+        let localTempFilePath = '';
         try {
-            await this.verifyDownload(localTempFilePath);
+            localTempFilePath = await this.downloadFile(downloadUriPrefix, enginePackageFileName, 'Downloading Python Analysis Engine... ');
+            await this.verifyDownload(localTempFilePath, platformString);
             await this.unpackArchive(context.extensionPath, localTempFilePath);
         } catch (err) {
             this.output.appendLine('failed.');
             this.output.appendLine(err);
             throw new Error(err);
         } finally {
-            fs.unlink(localTempFilePath, noop);
+            if (localTempFilePath.length > 0) {
+                await this.fs.deleteFile(localTempFilePath);
+            }
         }
     }
 
-    private async downloadFile(): Promise<string> {
-        const platformString = await this.platformData.getPlatformName();
-        const remoteFileName = `${downloadBaseFileName}-${platformString}.${downloadVersion}${downloadFileExtension}`;
-        const uri = `${downloadUriPrefix}/${remoteFileName}`;
+    public async downloadPythiaModel(context: ExtensionContext): Promise<void> {
+        const modelFolder = path.join(context.extensionPath, 'analysis', 'Pythia', 'model');
+        const localPath = path.join(modelFolder, pythiaModelName);
+        if (await this.fs.fileExists(localPath)) {
+            return;
+        }
+
+        let localTempFilePath = '';
+        try {
+            localTempFilePath = await this.downloadFile(downloadUriPrefix, pythiaModelName, 'Downloading IntelliSense Model File... ');
+            await this.fs.createDirectory(modelFolder);
+            await this.fs.copyFile(localTempFilePath, localPath);
+        } catch (err) {
+            this.output.appendLine('failed.');
+            this.output.appendLine(err);
+            throw new Error(err);
+        } finally {
+            if (localTempFilePath.length > 0) {
+                await this.fs.deleteFile(localTempFilePath);
+            }
+        }
+    }
+
+    private async downloadFile(location: string, fileName: string, title: string): Promise<string> {
+        const uri = `${location}/${fileName}`;
         this.output.append(`Downloading ${uri}... `);
         const tempFile = await createTemporaryFile(downloadFileExtension);
 
         const deferred = createDeferred();
-        const fileStream = fs.createWriteStream(tempFile.filePath);
+        const fileStream = fileSystem.createWriteStream(tempFile.filePath);
         fileStream.on('finish', () => {
             fileStream.close();
         }).on('error', (err) => {
             tempFile.cleanupCallback();
             deferred.reject(err);
         });
 
-        const title = 'Downloading Python Analysis Engine... ';
         await window.withProgress({
             location: ProgressLocation.Window,
             title
@@ -94,11 +122,11 @@ export class AnalysisEngineDownloader {
         return tempFile.filePath;
     }
 
-    private async verifyDownload(filePath: string): Promise<void> {
+    private async verifyDownload(filePath: string, platformString: string): Promise<void> {
         this.output.appendLine('');
         this.output.append('Verifying download... ');
         const verifier = new HashVerifier();
-        if (!await verifier.verifyHash(filePath, await this.platformData.getExpectedHash())) {
+        if (!await verifier.verifyHash(filePath, platformString, await this.platformData.getExpectedHash())) {
             throw new Error('Hash of the downloaded file does not match.');
         }
         this.output.append('valid.');
@@ -123,10 +151,10 @@ export class AnalysisEngineDownloader {
 
             let totalFiles = 0;
             let extractedFiles = 0;
-            zip.on('ready', () => {
+            zip.on('ready', async () => {
                 totalFiles = zip.entriesCount;
-                if (!fs.existsSync(installFolder)) {
-                    fs.mkdirSync(installFolder);
+                if (!await this.fs.directoryExists(installFolder)) {
+                    await this.fs.createDirectory(installFolder);
                 }
                 zip.extract(null, installFolder, (err, count) => {
                     if (err) {
@@ -147,7 +175,7 @@ export class AnalysisEngineDownloader {
         // Set file to executable
         if (!this.platform.isWindows) {
             const executablePath = path.join(installFolder, this.platformData.getEngineExecutableName());
-            fs.chmodSync(executablePath, '0764'); // -rwxrw-r--
+            fileSystem.chmodSync(executablePath, '0764'); // -rwxrw-r--
         }
     }
 }

src/client/activation/hashVerifier.ts

@@ -6,7 +6,7 @@ import * as fs from 'fs';
 import { createDeferred } from '../common/helpers';
 
 export class HashVerifier {
-    public async verifyHash(filePath: string, expectedDigest: string): Promise<boolean> {
+    public async verifyHash(filePath: string, platformString: string, expectedDigest: string): Promise<boolean> {
         const readStream = fs.createReadStream(filePath);
         const deferred = createDeferred();
         const hash = createHash('sha512');
@@ -23,6 +23,6 @@ export class HashVerifier {
         readStream.pipe(hash);
         await deferred.promise;
         const actual = hash.read();
-        return expectedDigest === '' ? true : actual === expectedDigest;
+        return expectedDigest === platformString ? true : actual === expectedDigest;
     }
 }