Changes to IntelliCode and engine downloads (#1715)

Mikhail Arkhipov · web-flow · commit ca657e0b54fa · 2018-05-21T15:11:07.000-07:00
* Undo changes * Test fixes * Increase timeout * Remove double event listening * Remove test * Revert "Remove test" This reverts commit e240c3f. * Revert "Remove double event listening" This reverts commit af573be. * #1096 The if statement is automatically formatted incorrectly * Merge fix * Add more tests * More tests * Typo * Test * Also better handle multiline arguments * Add a couple missing periods [skip ci] * Undo changes * Test fixes * Increase timeout * Remove double event listening * Remove test * Revert "Remove test" This reverts commit e240c3f. * Revert "Remove double event listening" This reverts commit af573be. * Merge fix * #1257 On type formatting errors for args and kwargs * Handle f-strings * Stop importing from test code * #1308 Single line statements leading to an indentation on the next line * #726 editing python after inline if statement invalid indent * Undo change * Move constant * Harden LS startup error checks * #1364 Intellisense doesn't work after specific const string * Telemetry for the analysis enging * PR feedback * Fix typo * Test baseline update * Jedi 0.12 * Priority to goto_defition * News * Replace unzip * Linux flavors + test * Grammar check * Grammar test * Test baselines * Add news * Pin dependency [skip ci] * Specify markdown as preferable format * Improve function argument detection * Specify markdown * Pythia setting * Baseline updates * Baseline update * Improve startup * Handle missing interpreter better * Handle interpreter change * Delete old file * Fix LS startup time reporting * Remove Async suffix from IFileSystem * Remove Pythia * Remove pre-packaged MSIL * Exe name on Unix * Plain linux * Fix casing * Fix message * Update PTVS engine activation steps
diff --git a/CONTRIBUTING - PYTHON_ANALYSIS.md b/CONTRIBUTING - PYTHON_ANALYSIS.md
@@ -32,6 +32,7 @@ Visual Studio 2017:
 3. Binaries arrive in *Python/BuildOutput/VsCode/raw*
 4. Delete contents of the *analysis* folder in the Python Extension folder
 5. Copy *.dll, *.pdb, *.json fron *Python/BuildOutput/VsCode/raw* to *analysis*
+6. In VS Code set setting *python.downloadCodeAnalysis* to *false*
 
 ### Debugging code in Python Extension to VS Code
 Folow regular TypeScript debugging steps
diff --git a/package.json b/package.json
@@ -1232,12 +1232,6 @@
                     "description": "Whether to install Python modules globally when not using an environment.",
                     "scope": "resource"
                 },
-                "python.pythiaEnabled": {
-                    "type": "boolean",
-                    "default": true,
-                    "description": "Enables AI-driven additions to the completion list. Does not apply to Jedi.",
-                    "scope": "resource"
-                },
                 "python.jediEnabled": {
                     "type": "boolean",
                     "default": true,
diff --git a/src/client/activation/analysis.ts b/src/client/activation/analysis.ts
@@ -9,7 +9,6 @@ import { IApplicationShell } from '../common/application/types';
 import { isTestExecution, STANDARD_OUTPUT_CHANNEL } from '../common/constants';
 import { createDeferred, Deferred } from '../common/helpers';
 import { IFileSystem, IPlatformService } from '../common/platform/types';
-import { IProcessServiceFactory } from '../common/process/types';
 import { StopWatch } from '../common/stopWatch';
 import { IConfigurationService, IOutputChannel, IPythonSettings } from '../common/types';
 import { IEnvironmentVariablesProvider } from '../common/variables/types';
@@ -103,32 +102,19 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
         // Determine if we are running MSIL/Universal via dotnet or self-contained app.
         const mscorlib = path.join(context.extensionPath, analysisEngineFolder, 'mscorlib.dll');
         const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
-        let downloadPackage = false;
 
         const reporter = getTelemetryReporter();
         reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ENABLED);
 
-        await this.checkPythiaModel(context, downloader);
-
-        if (!await this.fs.fileExists(mscorlib)) {
-            // Depends on .NET Runtime or SDK
+        const settings = this.configuration.getSettings();
+        if (!settings.downloadCodeAnalysis) {
+            // Depends on .NET Runtime or SDK. Typically development-only case.
             this.languageClient = this.createSimpleLanguageClient(context, clientOptions);
-            try {
-                await this.tryStartLanguageClient(context, this.languageClient);
-                return true;
-            } catch (ex) {
-                if (await this.isDotNetInstalled()) {
-                    this.appShell.showErrorMessage(`.NET Runtime appears to be installed but the language server did not start. Error ${ex}`);
-                    reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ERROR, { error: 'Failed to start (MSIL)' });
-                    return false;
-                }
-                // No .NET Runtime, no mscorlib - need to download self-contained package.
-                downloadPackage = true;
-            }
+            await this.tryStartLanguageClient(context, this.languageClient);
+            return true;
         }
 
-        if (downloadPackage) {
-            this.appShell.showWarningMessage('.NET Runtime is not found, platform-specific Python Analysis Engine will be downloaded.');
+        if (!await this.fs.fileExists(mscorlib)) {
             await downloader.downloadAnalysisEngine(context);
             reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_DOWNLOADED);
         }
@@ -254,22 +240,9 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
                     maxDocumentationTextLength: 0
                 },
                 asyncStartup: true,
-                pythiaEnabled: settings.pythiaEnabled,
+                intelliCodeEnabled: settings.intelliCodeEnabled,
                 testEnvironment: isTestExecution()
             }
         };
     }
-
-    private async isDotNetInstalled(): Promise<boolean> {
-        const ps = await this.services.get<IProcessServiceFactory>(IProcessServiceFactory).create();
-        const result = await ps.exec('dotnet', ['--version']).catch(() => { return { stdout: '' }; });
-        return result.stdout.trim().startsWith('2.');
-    }
-
-    private async checkPythiaModel(context: ExtensionContext, downloader: AnalysisEngineDownloader): Promise<void> {
-        const settings = this.configuration.getSettings();
-        if (settings.pythiaEnabled) {
-            await downloader.downloadPythiaModel(context);
-        }
-    }
 }
diff --git a/src/client/activation/analysisEngineHashes.ts b/src/client/activation/analysisEngineHashes.ts
@@ -7,10 +7,4 @@
 export const analysis_engine_win_x86_sha512 = 'win-x86';
 export const analysis_engine_win_x64_sha512 = 'win-x64';
 export const analysis_engine_osx_x64_sha512 = 'osx-x64';
-export const analysis_engine_centos_x64_sha512 = 'centos-x64';
-export const analysis_engine_debian_x64_sha512 = 'debian-x64';
-export const analysis_engine_fedora_x64_sha512 = 'fedora-x64';
-export const analysis_engine_ol_x64_sha512 = 'ol-x64';
-export const analysis_engine_opensuse_x64_sha512 = 'opensuse-x64';
-export const analysis_engine_rhel_x64_sha512 = 'rhel-x64';
-export const analysis_engine_ubuntu_x64_sha512 = 'ubuntu-x64';
+export const analysis_engine_linux_x64_sha512 = 'linux-x64';
diff --git a/src/client/activation/downloader.ts b/src/client/activation/downloader.ts
@@ -18,10 +18,10 @@ import { PlatformData } from './platformData';
 const StreamZip = require('node-stream-zip');
 
 const downloadUriPrefix = 'https://pvsc.blob.core.windows.net/python-analysis';
-const downloadBaseFileName = 'python-analysis-vscode';
+const downloadBaseFileName = 'Python-Analysis-VSCode';
 const downloadVersion = '0.1.0';
 const downloadFileExtension = '.nupkg';
-const pythiaModelName = 'model-sequence.json.gz';
+const modelName = 'model-sequence.json.gz';
 
 export class AnalysisEngineDownloader {
     private readonly output: OutputChannel;
@@ -56,16 +56,16 @@ export class AnalysisEngineDownloader {
         }
     }
 
-    public async downloadPythiaModel(context: ExtensionContext): Promise<void> {
+    public async downloadIntelliCodeModel(context: ExtensionContext): Promise<void> {
         const modelFolder = path.join(context.extensionPath, 'analysis', 'Pythia', 'model');
-        const localPath = path.join(modelFolder, pythiaModelName);
+        const localPath = path.join(modelFolder, modelName);
         if (await this.fs.fileExists(localPath)) {
             return;
         }
 
         let localTempFilePath = '';
         try {
-            localTempFilePath = await this.downloadFile(downloadUriPrefix, pythiaModelName, 'Downloading IntelliSense Model File... ');
+            localTempFilePath = await this.downloadFile(downloadUriPrefix, modelName, 'Downloading IntelliCode Model File... ');
             await this.fs.createDirectory(modelFolder);
             await this.fs.copyFile(localTempFilePath, localPath);
         } catch (err) {
@@ -129,11 +129,10 @@ export class AnalysisEngineDownloader {
         if (!await verifier.verifyHash(filePath, platformString, await this.platformData.getExpectedHash())) {
             throw new Error('Hash of the downloaded file does not match.');
         }
-        this.output.append('valid.');
+        this.output.appendLine('valid.');
     }
 
     private async unpackArchive(extensionPath: string, tempFilePath: string): Promise<void> {
-        this.output.appendLine('');
         this.output.append('Unpacking archive... ');
 
         const installFolder = path.join(extensionPath, this.engineFolder);
@@ -170,12 +169,12 @@ export class AnalysisEngineDownloader {
             });
             return deferred.promise;
         });
-        this.output.append('done.');
 
         // Set file to executable
         if (!this.platform.isWindows) {
             const executablePath = path.join(installFolder, this.platformData.getEngineExecutableName());
             fileSystem.chmodSync(executablePath, '0764'); // -rwxrw-r--
         }
+        this.output.appendLine('done.');
     }
 }
diff --git a/src/client/activation/platformData.ts b/src/client/activation/platformData.ts
@@ -3,31 +3,14 @@
 
 import { IFileSystem, IPlatformService } from '../common/platform/types';
 import {
-    analysis_engine_centos_x64_sha512,
-    analysis_engine_debian_x64_sha512,
-    analysis_engine_fedora_x64_sha512,
-    analysis_engine_ol_x64_sha512,
-    analysis_engine_opensuse_x64_sha512,
+    analysis_engine_linux_x64_sha512,
     analysis_engine_osx_x64_sha512,
-    analysis_engine_rhel_x64_sha512,
-    analysis_engine_ubuntu_x64_sha512,
     analysis_engine_win_x64_sha512,
     analysis_engine_win_x86_sha512
 } from './analysisEngineHashes';
 
-// '/etc/os-release', ID=flavor
-const supportedLinuxFlavors = [
-    'centos',
-    'debian',
-    'fedora',
-    'ol',
-    'opensuse',
-    'rhel',
-    'ubuntu'
-];
-
 export class PlatformData {
-    constructor(private platform: IPlatformService, private fs: IFileSystem) { }
+    constructor(private platform: IPlatformService, fs: IFileSystem) { }
     public async getPlatformName(): Promise<string> {
         if (this.platform.isWindows) {
             return this.platform.is64bit ? 'win-x64' : 'win-x86';
@@ -39,14 +22,7 @@ export class PlatformData {
             if (!this.platform.is64bit) {
                 throw new Error('Python Analysis Engine does not support 32-bit Linux.');
             }
-            const linuxFlavor = await this.getLinuxFlavor();
-            if (linuxFlavor.length === 0) {
-                throw new Error('Unable to determine Linux flavor from /etc/os-release.');
-            }
-            if (supportedLinuxFlavors.indexOf(linuxFlavor) < 0) {
-                throw new Error(`${linuxFlavor} is not supported.`);
-            }
-            return `${linuxFlavor}-x64`;
+            return 'linux-x64';
         }
         throw new Error('Unknown OS platform.');
     }
@@ -58,7 +34,7 @@ export class PlatformData {
     public getEngineExecutableName(): string {
         return this.platform.isWindows
             ? 'Microsoft.PythonTools.VsCode.exe'
-            : 'Microsoft.PythonTools.VsCode';
+            : 'Microsoft.PythonTools.VsCode.VsCode';
     }
 
     public async getExpectedHash(): Promise<string> {
@@ -69,30 +45,8 @@ export class PlatformData {
             return analysis_engine_osx_x64_sha512;
         }
         if (this.platform.isLinux && this.platform.is64bit) {
-            const linuxFlavor = await this.getLinuxFlavor();
-            // tslint:disable-next-line:switch-default
-            switch (linuxFlavor) {
-                case 'centos': return analysis_engine_centos_x64_sha512;
-                case 'debian': return analysis_engine_debian_x64_sha512;
-                case 'fedora': return analysis_engine_fedora_x64_sha512;
-                case 'ol': return analysis_engine_ol_x64_sha512;
-                case 'opensuse': return analysis_engine_opensuse_x64_sha512;
-                case 'rhel': return analysis_engine_rhel_x64_sha512;
-                case 'ubuntu': return analysis_engine_ubuntu_x64_sha512;
-            }
+            return analysis_engine_linux_x64_sha512;
         }
         throw new Error('Unknown platform.');
     }
-
-    private async getLinuxFlavor(): Promise<string> {
-        const verFile = '/etc/os-release';
-        const data = await this.fs.readFile(verFile);
-        if (data) {
-            const res = /ID=(.*)/.exec(data);
-            if (res && res.length > 1) {
-                return res[1];
-            }
-        }
-        return '';
-    }
 }
diff --git a/src/client/common/configSettings.ts b/src/client/common/configSettings.ts
@@ -25,7 +25,8 @@ export const IS_WINDOWS = /^win/.test(process.platform);
 // tslint:disable-next-line:completed-docs
 export class PythonSettings extends EventEmitter implements IPythonSettings {
     private static pythonSettings: Map<string, PythonSettings> = new Map<string, PythonSettings>();
-    public pythiaEnabled = true;
+    public intelliCodeEnabled = true;
+    public downloadCodeAnalysis = true;
     public jediEnabled = true;
     public jediPath = '';
     public jediMemoryLimit = 1024;
@@ -115,6 +116,7 @@ export class PythonSettings extends EventEmitter implements IPythonSettings {
         this.venvPath = systemVariables.resolveAny(pythonSettings.get<string>('venvPath'))!;
         this.venvFolders = systemVariables.resolveAny(pythonSettings.get<string[]>('venvFolders'))!;
 
+        this.downloadCodeAnalysis = systemVariables.resolveAny(pythonSettings.get<boolean>('downloadCodeAnalysis', true))!;
         this.jediEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('jediEnabled', true))!;
         if (this.jediEnabled) {
             // tslint:disable-next-line:no-backbone-get-set-outside-model no-non-null-assertion
@@ -126,7 +128,7 @@ export class PythonSettings extends EventEmitter implements IPythonSettings {
             }
             this.jediMemoryLimit = pythonSettings.get<number>('jediMemoryLimit')!;
         } else {
-            this.pythiaEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('pythiaEnabled', true))!;
+            this.intelliCodeEnabled = systemVariables.resolveAny(pythonSettings.get<boolean>('intelliCodeEnabled', true))!;
         }
 
         // tslint:disable-next-line:no-backbone-get-set-outside-model no-non-null-assertion
diff --git a/src/client/common/types.ts b/src/client/common/types.ts
@@ -100,7 +100,8 @@ export interface IPythonSettings {
     readonly pythonPath: string;
     readonly venvPath: string;
     readonly venvFolders: string[];
-    readonly pythiaEnabled: boolean;
+    readonly intelliCodeEnabled: boolean;
+    readonly downloadCodeAnalysis: boolean;
     readonly jediEnabled: boolean;
     readonly jediPath: string;
     readonly jediMemoryLimit: number;
