Use new environment variable parser

src/client/common/process/proc.ts

@@ -19,6 +19,10 @@ export class ProcessService implements IProcessService {
         if (!spawnOptions.env || Object.keys(spawnOptions).length === 0) {
             spawnOptions.env = process.env;
         }
+
+        // Always ensure we have unbuffered output.
+        spawnOptions.env.PYTHONUNBUFFERED = '1';
+
         const proc = spawn(file, args, spawnOptions);
         let procExited = false;
 
@@ -72,6 +76,10 @@ export class ProcessService implements IProcessService {
         if (!spawnOptions.env || Object.keys(spawnOptions).length === 0) {
             spawnOptions.env = process.env;
         }
+
+        // Always ensure we have unbuffered output.
+        spawnOptions.env.PYTHONUNBUFFERED = '1';
+
         const proc = spawn(file, args, spawnOptions);
         const deferred = createDeferred<ExecutionResult<string>>();
         const disposables: Disposable[] = [];

src/client/common/variables/environment.ts

@@ -11,7 +11,7 @@ import { EnvironmentVariables, IEnvironmentVariablesService } from './types';
 
 @injectable()
 export class EnvironmentVariablesService implements IEnvironmentVariablesService {
-    constructor( @inject(IsWindows) private isWidows: boolean) { }
+    constructor( @inject(IsWindows) private isWindows: boolean) { }
     public async parseFile(filePath: string): Promise<EnvironmentVariables | undefined> {
         const exists = await fs.pathExists(filePath);
         if (!exists) {
@@ -27,7 +27,7 @@ export class EnvironmentVariablesService implements IEnvironmentVariablesService
                     return resolve(undefined);
                 }
                 this.appendPythonPath(vars, process.env.PYTHONPATH);
-                const pathVariable = this.isWidows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
+                const pathVariable = this.isWindows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
                 this.appendPath(vars, process.env[pathVariable]);
                 resolve(vars);
             });
@@ -37,7 +37,7 @@ export class EnvironmentVariablesService implements IEnvironmentVariablesService
         if (!target) {
             return;
         }
-        const pathVariable = this.isWidows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
+        const pathVariable = this.isWindows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
         const settingsNotToMerge = ['PYTHONPATH', pathVariable];
         Object.keys(source).forEach(setting => {
             if (settingsNotToMerge.indexOf(setting) >= 0) {
@@ -55,11 +55,11 @@ export class EnvironmentVariablesService implements IEnvironmentVariablesService
         return this.appendOrPrependPaths(vars, 'PYTHONPATH', true, ...pythonPaths);
     }
     public prependPath(vars: EnvironmentVariables, ...paths: string[]) {
-        const pathVariable = this.isWidows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
+        const pathVariable = this.isWindows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
         return this.appendOrPrependPaths(vars, pathVariable, false, ...paths);
     }
     public appendPath(vars: EnvironmentVariables, ...paths: string[]) {
-        const pathVariable = this.isWidows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
+        const pathVariable = this.isWindows ? WINDOWS_PATH_VARIABLE_NAME : NON_WINDOWS_PATH_VARIABLE_NAME;
         return this.appendOrPrependPaths(vars, pathVariable, true, ...paths);
     }
     private appendOrPrependPaths(vars: EnvironmentVariables, variableName: 'PATH' | 'Path' | 'PYTHONPATH', append: boolean, ...pythonPaths: string[]) {

src/client/debugger/Common/Utils.ts

@@ -4,8 +4,7 @@ import * as child_process from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as untildify from 'untildify';
-import { mergeEnvVariables, mergePythonPath, parseEnvFile } from '../../common/envFileParser';
-import { IPythonEvaluationResult, IPythonModule, IPythonProcess, IPythonThread } from './Contracts';
+import { IPythonModule, IPythonProcess, IPythonThread } from './Contracts';
 
 export const IS_WINDOWS = /^win/.test(process.platform);
 export const PATH_VARIABLE_NAME = IS_WINDOWS ? 'Path' : 'PATH';
@@ -31,7 +30,7 @@ export function validatePathSync(filePath: string): boolean {
         return false;
     }
     if (PathValidity.has(filePath)) {
-        return PathValidity.get(filePath);
+        return PathValidity.get(filePath)!;
     }
     const exists = fs.existsSync(filePath);
     PathValidity.set(filePath, exists);
@@ -116,30 +115,3 @@ function isValidPythonPath(pythonPath): boolean {
         return false;
     }
 }
-
-type EnvVars = Object & { [key: string]: string };
-
-export function getCustomEnvVars(envVars: Object, envFile: string, mergeWithProcessEnvVars: boolean = true): EnvVars {
-    let envFileVars: EnvVars = null;
-    if (typeof envFile === 'string' && envFile.length > 0 && fs.existsSync(envFile)) {
-        try {
-            envFileVars = parseEnvFile(envFile, mergeWithProcessEnvVars);
-        } catch (ex) {
-            console.error('Failed to load env file');
-            console.error(ex);
-        }
-    }
-    if (envFileVars && Object.keys(envFileVars).length > 0) {
-        if (!envVars || Object.keys(envVars).length === 0) {
-            return envFileVars;
-        } else {
-            envVars = envVars || {};
-            return mergeEnvVariables(envVars as EnvVars, envFileVars);
-        }
-    }
-    if (!envVars || Object.keys(envVars).length === 0) {
-        return null;
-    }
-
-    return mergePythonPath(envVars as EnvVars, process.env.PYTHONPATH);
-}

src/client/debugger/DebugClients/LocalDebugClient.ts

@@ -4,9 +4,11 @@ import * as path from 'path';
 import { DebugSession, OutputEvent } from 'vscode-debugadapter';
 import { DebugProtocol } from 'vscode-debugprotocol';
 import { open } from '../../common/open';
+import { EnvironmentVariablesService } from '../../common/variables/environment';
+import { EnvironmentVariables } from '../../common/variables/types';
 import { IDebugServer, IPythonProcess } from '../Common/Contracts';
 import { LaunchRequestArguments } from '../Common/Contracts';
-import { getCustomEnvVars } from '../Common/Utils';
+import { IS_WINDOWS } from '../Common/Utils';
 import { BaseDebugServer } from '../DebugServers/BaseDebugServer';
 import { LocalDebugServer } from '../DebugServers/LocalDebugServer';
 import { DebugClient, DebugType } from './DebugClient';
@@ -18,9 +20,9 @@ const VALID_DEBUG_OPTIONS = [
     'DjangoDebugging'];
 
 export class LocalDebugClient extends DebugClient {
-    protected pyProc: child_process.ChildProcess;
+    protected pyProc: child_process.ChildProcess | undefined;
     protected pythonProcess: IPythonProcess;
-    protected debugServer: BaseDebugServer;
+    protected debugServer: BaseDebugServer | undefined;
     // tslint:disable-next-line:no-any
     constructor(args: any, debugSession: DebugSession, private canLaunchTerminal: boolean) {
         super(args, debugSession);
@@ -38,24 +40,24 @@ export class LocalDebugClient extends DebugClient {
 
     public Stop() {
         if (this.debugServer) {
-            this.debugServer.Stop();
-            this.debugServer = null;
+            this.debugServer!.Stop();
+            this.debugServer = undefined;
         }
 
         if (this.pyProc) {
             try {
-                this.pyProc.send('EXIT');
+                this.pyProc!.send('EXIT');
                 // tslint:disable-next-line:no-empty
             } catch { }
             try {
-                this.pyProc.stdin.write('EXIT');
+                this.pyProc!.stdin.write('EXIT');
                 // tslint:disable-next-line:no-empty
             } catch { }
             try {
-                this.pyProc.disconnect();
+                this.pyProc!.disconnect();
                 // tslint:disable-next-line:no-empty
             } catch { }
-            this.pyProc = null;
+            this.pyProc = undefined;
         }
     }
     protected getLauncherFilePath(): string {
@@ -71,7 +73,8 @@ export class LocalDebugClient extends DebugClient {
         }
     }
     // tslint:disable-next-line:max-func-body-length member-ordering no-any
-    public LaunchApplicationToDebug(dbgServer: IDebugServer, processErrored: (error: any) => void): Promise<any> {
+    public async LaunchApplicationToDebug(dbgServer: IDebugServer, processErrored: (error: any) => void): Promise<any> {
+        const environmentVariables = await this.getEnvironmentVariables();
         // tslint:disable-next-line:max-func-body-length cyclomatic-complexity no-any
         return new Promise<any>((resolve, reject) => {
             const fileDir = this.args && this.args.program ? path.dirname(this.args.program) : '';
@@ -83,8 +86,6 @@ export class LocalDebugClient extends DebugClient {
             if (typeof this.args.pythonPath === 'string' && this.args.pythonPath.trim().length > 0) {
                 pythonPath = this.args.pythonPath;
             }
-            let environmentVariables = getCustomEnvVars(this.args.env, this.args.envFile, false);
-            environmentVariables = environmentVariables ? environmentVariables : {};
             if (!environmentVariables.hasOwnProperty('PYTHONIOENCODING')) {
                 environmentVariables.PYTHONIOENCODING = 'UTF-8';
             }
@@ -103,12 +104,16 @@ export class LocalDebugClient extends DebugClient {
                     break;
                 }
                 default: {
+                    // As we're spawning the process, we need to ensure all env variables are passed.
+                    // Including those from the current process (i.e. everything, not just custom vars).
+                    const envParser = new EnvironmentVariablesService(IS_WINDOWS);
+                    envParser.mergeVariables(process.env as EnvironmentVariables, environmentVariables);
                     this.pyProc = child_process.spawn(pythonPath, args, { cwd: processCwd, env: environmentVariables });
-                    this.handleProcessOutput(this.pyProc, reject);
+                    this.handleProcessOutput(this.pyProc!, reject);
 
                     // Here we wait for the application to connect to the socket server.
                     // Only once connected do we know that the application has successfully launched.
-                    this.debugServer.DebugClientConnected
+                    this.debugServer!.DebugClientConnected
                         .then(resolve)
                         .catch(ex => console.error('Python Extension: debugServer.DebugClientConnected', ex));
                 }
@@ -120,10 +125,10 @@ export class LocalDebugClient extends DebugClient {
         proc.on('error', error => {
             // If debug server has started, then don't display errors.
             // The debug adapter will get this info from the debugger (e.g. ptvsd lib).
-            if (!this.debugServer && this.debugServer.IsRunning) {
+            if (!this.debugServer && this.debugServer!.IsRunning) {
                 return;
             }
-            if (!this.debugServer.IsRunning && typeof (error) === 'object' && error !== null) {
+            if (!this.debugServer && !this.debugServer!.IsRunning && typeof (error) === 'object' && error !== null) {
                 return failedToLaunch(error);
             }
             // This could happen when the debugger didn't launch at all, e.g. python doesn't exist.
@@ -136,7 +141,7 @@ export class LocalDebugClient extends DebugClient {
 
             // Either way, we need some code in here so we read the stdout of the python process,
             // Else it just keep building up (related to issue #203 and #52).
-            if (this.debugServer && !this.debugServer.IsRunning) {
+            if (this.debugServer && !this.debugServer!.IsRunning) {
                 return failedToLaunch(error);
             }
         });
@@ -193,12 +198,32 @@ export class LocalDebugClient extends DebugClient {
                     this.pyProc = proc;
                     resolve();
                 }, error => {
-                    if (!this.debugServer && this.debugServer.IsRunning) {
+                    if (!this.debugServer && this.debugServer!.IsRunning) {
                         return;
                     }
                     reject(error);
                 });
             }
         });
     }
+    private async getEnvironmentVariables(): Promise<EnvironmentVariables> {
+        const args = this.args as LaunchRequestArguments;
+        const envParser = new EnvironmentVariablesService(IS_WINDOWS);
+        const envFileVars = await envParser.parseFile(args.envFile);
+
+        const hasEnvVars = args.env && Object.keys(args.env).length > 0;
+        if (!envFileVars && !hasEnvVars) {
+            return {};
+        }
+        if (envFileVars && !hasEnvVars) {
+            return envFileVars!;
+        }
+        if (!envFileVars && hasEnvVars) {
+            return args.env as EnvironmentVariables;
+        }
+        // Merge the two sets of environment variables.
+        const env = { ...args.env } as EnvironmentVariables;
+        envParser.mergeVariables(envFileVars!, env);
+        return env;
+    }
 }

src/client/debugger/DebugClients/NonDebugClient.ts

@@ -1,10 +1,7 @@
 import { ChildProcess } from 'child_process';
 import * as path from 'path';
-import { DebugSession, OutputEvent } from 'vscode-debugadapter';
-import { IPythonProcess } from '../Common/Contracts';
+import { DebugSession } from 'vscode-debugadapter';
 import { LaunchRequestArguments } from '../Common/Contracts';
-import { BaseDebugServer } from '../DebugServers/BaseDebugServer';
-import { LocalDebugServer } from '../DebugServers/LocalDebugServer';
 import { DebugType } from './DebugClient';
 import { LocalDebugClient } from './LocalDebugClient';
 
@@ -22,10 +19,10 @@ export class NonDebugClient extends LocalDebugClient {
         super.Stop();
         if (this.pyProc) {
             try {
-                this.pyProc.kill();
+                this.pyProc!.kill();
                 // tslint:disable-next-line:no-empty
             } catch { }
-            this.pyProc = null;
+            this.pyProc = undefined;
         }
     }
     protected handleProcessOutput(proc: ChildProcess, _failedToLaunch: (error: Error | string | Buffer) => void) {

src/client/interpreter/configuration/pythonPathUpdaterService.ts

@@ -3,6 +3,7 @@ import { ConfigurationTarget, Uri, window } from 'vscode';
 import { sendTelemetryEvent } from '../../telemetry';
 import { PYTHON_INTERPRETER } from '../../telemetry/constants';
 import { StopWatch } from '../../telemetry/stopWatch';
+import { PythonInterpreterTelemetry } from '../../telemetry/types';
 import { IInterpreterVersionService } from '../interpreterVersion';
 import { IPythonPathUpdaterServiceFactory } from './types';
 
@@ -27,20 +28,24 @@ export class PythonPathUpdaterService {
             .catch(ex => console.error('Python Extension: sendTelemetry', ex));
     }
     private async sendTelemetry(duration: number, failed: boolean, trigger: 'ui' | 'shebang' | 'load', pythonPath: string) {
-        let version: string | undefined;
-        let pipVersion: string | undefined;
+        const telemtryProperties: PythonInterpreterTelemetry = {
+            failed, trigger
+        };
         if (!failed) {
             const pyVersionPromise = this.interpreterVersionService.getVersion(pythonPath, '')
                 .then(pyVersion => pyVersion.length === 0 ? undefined : pyVersion);
             const pipVersionPromise = this.interpreterVersionService.getPipVersion(pythonPath)
                 .then(value => value.length === 0 ? undefined : value)
                 .catch(() => undefined);
             const versions = await Promise.all([pyVersionPromise, pipVersionPromise]);
-            version = versions[0];
-            // tslint:disable-next-line:prefer-type-cast
-            pipVersion = versions[1] as string;
+            if (versions[0]) {
+                telemtryProperties.version = versions[0] as string;
+            }
+            if (versions[1]) {
+                telemtryProperties.pipVersion = versions[1] as string;
+            }
         }
-        sendTelemetryEvent(PYTHON_INTERPRETER, duration, { failed, trigger, version, pipVersion });
+        sendTelemetryEvent(PYTHON_INTERPRETER, duration, telemtryProperties);
     }
     private getPythonUpdaterService(configTarget: ConfigurationTarget, wkspace?: Uri) {
         switch (configTarget) {

src/client/telemetry/types.ts

@@ -18,8 +18,8 @@ export type LintingTelemetry = {
 export type PythonInterpreterTelemetry = {
     trigger: 'ui' | 'shebang' | 'load';
     failed: boolean;
-    version: string;
-    pipVersion: string;
+    version?: string;
+    pipVersion?: string;
 };
 export type CodeExecutionTelemetry = {
     scope: 'file' | 'selection';