Fix issues when running without debugging and debugged code terminates (#249)

- Fixes #25, #32, #35, #235, #242 (unable to run without debugging using CTRL+F5)   
- Fixes #191, #158, #24, #136  (error message displayed when debugged code terminates)  
- Fixes #157 (debugger crashes when python program does not launch)
- Fixes #114, #149, #250 (use vscode infrastructure to launch debugger in integrated and external terminals)  
- Fixes #239 Remove prompt added to pause terminal upon program completion

Author: DonJayamanne

package.json

@@ -0,0 +1,146 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""Run a block of code or Python file."""
+
+import sys
+import os.path
+import traceback
+import time
+import socket
+try:
+    import visualstudio_py_util as _vspu
+except:
+    traceback.print_exc()
+    print("""Internal error detected. Please copy the above traceback and report at
+https://github.com/Microsoft/vscode-python/issues""")
+    sys.exit(1)
+
+LAST = _vspu.to_bytes('LAST')
+OUTP = _vspu.to_bytes('OUTP')
+LOAD = _vspu.to_bytes('LOAD')
+
+def parse_argv():
+    """Parses arguments for use with the launcher.
+    Arguments are:
+    1. Working directory.
+    2. VS debugger port to connect to.
+    3. GUID for the debug session.
+    4. Debug options (not used).
+    5. '-m' or '-c' to override the default run-as mode. [optional].
+    6. Startup script name.
+    7. Script arguments.
+    """
+
+    # Change to directory we expected to start from.
+    os.chdir(sys.argv[1])
+
+    port_num = int(sys.argv[2])
+    debug_id = sys.argv[3]
+
+    del sys.argv[:5]
+
+    # Set run_as mode appropriately
+    run_as = 'script'
+    if sys.argv and sys.argv[0] == '-m':
+        run_as = 'module'
+        del sys.argv[0]
+    elif sys.argv and sys.argv[0] == '-c':
+        run_as = 'code'
+        del sys.argv[0]
+
+    # Preserve filename before we del sys.
+    filename = sys.argv[0]
+
+    # Fix sys.path to be the script file dir.
+    sys.path[0] = ''
+
+    pid = os.getpid()
+
+    return (filename, port_num, debug_id, pid, run_as)
+
+def run(file, port_num, debug_id, pid, run_as='script'):
+    attach_process(port_num, pid, debug_id)
+
+    # Now execute main file.
+    globals_obj = {'__name__': '__main__'}
+
+    try:
+        if run_as == 'module':
+            _vspu.exec_module(file, globals_obj)
+        elif run_as == 'code':
+            _vspu.exec_code(file, '<string>', globals_obj)
+        else:
+            _vspu.exec_file(file, globals_obj)
+    except:
+        exc_type, exc_value, exc_tb = sys.exc_info()
+        handle_exception(exc_type, exc_value, exc_tb)
+
+    _vspu.write_bytes(conn, LAST)
+    # Wait for message to be received by debugger.
+    time.sleep(0.5)
+
+
+def attach_process(port_num, pid, debug_id):
+    global conn
+    for i in xrange(50):
+        try:
+            conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            conn.connect(('127.0.0.1', port_num))
+            # Initial handshake.
+            _vspu.write_string(conn, debug_id)
+            _vspu.write_int(conn, 0)
+            _vspu.write_int(conn, pid)
+
+            # Notify debugger that process has launched.
+            _vspu.write_bytes(conn, LOAD)
+            _vspu.write_int(conn, 0)
+            break
+        except:
+            time.sleep(50./1000)
+    else:
+        raise Exception('failed to attach')
+
+
+def handle_exception(exc_type, exc_value, exc_tb):
+    # Specifies list of files not to display in stack trace.
+    do_not_debug = [__file__, _vspu.__file__]
+    if sys.version_info >= (3, 3):
+        do_not_debug.append('<frozen importlib._bootstrap>')
+    if sys.version_info >= (3, 5):
+        do_not_debug.append('<frozen importlib._bootstrap_external>')
+
+    # Remove debugger frames from the top and bottom of the traceback.
+    tb = traceback.extract_tb(exc_tb)
+    for i in [0, -1]:
+        while tb:
+            frame_file = path.normcase(tb[i][0])
+            if not any(is_same_py_file(frame_file, f) for f in do_not_debug):
+                break
+            del tb[i]
+
+    # Print the traceback.
+    if tb:
+        sys.stderr.write('Traceback (most recent call last):')
+        for out in traceback.format_list(tb):
+            sys.stderr.write(out)
+            sys.stderr.flush()
+
+    # Print the exception.
+    for out in traceback.format_exception_only(exc_type, exc_value):
+        sys.stderr.write(out)
+        sys.stderr.flush()
+
+
+def is_same_py_file(file_1, file_2):
+    """Compares 2 filenames accounting for .pyc files."""
+    if file_1.endswith('.pyc') or file_1.endswith('.pyo'):
+        file_1 = file_1[:-1]
+    if file_2.endswith('.pyc') or file_2.endswith('.pyo'):
+        file_2 = file_2[:-1]
+
+    return file_1 == file_2
+
+if __name__ == '__main__':
+    filename, port_num, debug_id, pid, run_as = parse_argv()
+    run(filename, port_num, debug_id, pid, run_as)

pythonFiles/PythonTools/visualstudio_py_launcher_nodebug.py

@@ -1,7 +1,9 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-export function parseEnvFile(envFile: string): any {
+type EnvVars = Object & { [key: string]: string };
+
+export function parseEnvFile(envFile: string, mergeWithProcessEnvVars: boolean = true): EnvVars {
     const buffer = fs.readFileSync(envFile, 'utf8');
     const env = {};
     buffer.split('\n').forEach(line => {
@@ -14,28 +16,43 @@ export function parseEnvFile(envFile: string): any {
             env[r[1]] = value.replace(/(^['"]|['"]$)/g, '');
         }
     });
-    return mergeEnvVariables(env);
+    return mergeWithProcessEnvVars ? mergeEnvVariables(env, process.env) : mergePythonPath(env, process.env.PYTHONPATH);
 }
 
-export function mergeEnvVariables(newVariables: { [key: string]: string }, mergeWith: any = process.env): any {
-    for (let setting in mergeWith) {
-        if (setting === 'PYTHONPATH') {
-            let PYTHONPATH: string = newVariables['PYTHONPATH'];
-            if (typeof PYTHONPATH !== 'string') {
-                PYTHONPATH = '';
-            }
-            if (mergeWith['PYTHONPATH']) {
-                PYTHONPATH += (PYTHONPATH.length > 0 ? path.delimiter : '') + mergeWith['PYTHONPATH'];
-            }
-            if (PYTHONPATH.length > 0) {
-                newVariables[setting] = PYTHONPATH;
-            }
-            continue;
-        }
-        if (!newVariables[setting]) {
-            newVariables[setting] = mergeWith[setting];
+/**
+ * Merge the target environment variables into the source.
+ * Note: The source variables are modified and returned (i.e. it modifies value passed in).
+ * @export
+ * @param {EnvVars} targetEnvVars target environment variables.
+ * @param {EnvVars} [sourceEnvVars=process.env] source environment variables (defaults to current process variables).
+ * @returns {EnvVars}
+ */
+export function mergeEnvVariables(targetEnvVars: EnvVars, sourceEnvVars: EnvVars = process.env): EnvVars {
+    Object.keys(sourceEnvVars).forEach(setting => {
+        if (targetEnvVars[setting] === undefined) {
+            targetEnvVars[setting] = sourceEnvVars[setting];
         }
+    });
+    return mergePythonPath(targetEnvVars, sourceEnvVars.PYTHONPATH);
+}
+
+/**
+ * Merge the target PYTHONPATH value into the env variables passed.
+ * Note: The env variables passed in are modified and returned (i.e. it modifies value passed in).
+ * @export
+ * @param {EnvVars} env target environment variables.
+ * @param {string | undefined} [currentPythonPath] PYTHONPATH value.
+ * @returns {EnvVars}
+ */
+export function mergePythonPath(env: EnvVars, currentPythonPath: string | undefined): EnvVars {
+    if (typeof currentPythonPath !== 'string' || currentPythonPath.length === 0) {
+        return env;
     }
 
-    return newVariables;
+    if (typeof env.PYTHONPATH === 'string' && env.PYTHONPATH.length > 0) {
+        env.PYTHONPATH = env.PYTHONPATH + path.delimiter + currentPythonPath;
+    } else {
+        env.PYTHONPATH = currentPythonPath;
+    }
+    return env;
 }

src/client/common/envFileParser.ts

@@ -1,5 +1,6 @@
 "use strict";
 import * as net from "net";
+import { ChildProcess } from 'child_process';
 import { DebugProtocol } from "vscode-debugprotocol";
 import { OutputEvent } from "vscode-debugadapter";
 
@@ -49,7 +50,6 @@ export interface LaunchRequestArguments extends DebugProtocol.LaunchRequestArgum
     stopOnEntry?: boolean;
     args: string[];
     applicationType?: string;
-    externalConsole?: boolean;
     cwd?: string;
     debugOptions?: string[];
     env?: Object;
@@ -103,8 +103,9 @@ export enum PythonEvaluationResultFlags {
 }
 
 export interface IPythonProcess extends NodeJS.EventEmitter {
-    Connect(buffer: Buffer, socket: net.Socket, isRemoteProcess: boolean);
+    Connect(buffer: Buffer, socket: net.Socket, isRemoteProcess: boolean): boolean;
     HandleIncomingData(buffer: Buffer);
+    attach(proc: ChildProcess): void;
     Detach();
     Kill();
     SendStepInto(threadId: number);

src/client/debugger/Common/Contracts.ts

@@ -1,11 +1,11 @@
-"use strict";
+'use strict';
 
-import { IPythonProcess, IPythonThread, IPythonModule, IPythonEvaluationResult } from "./Contracts";
-import * as path from "path";
-import * as fs from 'fs';
 import * as child_process from 'child_process';
-import { mergeEnvVariables, parseEnvFile } from '../../common/envFileParser';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as untildify from 'untildify';
+import { mergeEnvVariables, mergePythonPath, parseEnvFile } from '../../common/envFileParser';
+import { IPythonEvaluationResult, IPythonModule, IPythonProcess, IPythonThread } from './Contracts';
 
 export const IS_WINDOWS = /^win/.test(process.platform);
 export const PATH_VARIABLE_NAME = IS_WINDOWS ? 'Path' : 'PATH';
@@ -38,7 +38,7 @@ export function validatePathSync(filePath: string): boolean {
     return exists;
 }
 
-export function CreatePythonThread(id: number, isWorker: boolean, process: IPythonProcess, name: string = ""): IPythonThread {
+export function CreatePythonThread(id: number, isWorker: boolean, process: IPythonProcess, name: string = ''): IPythonThread {
     return {
         IsWorkerThread: isWorker,
         Process: process,
@@ -50,15 +50,13 @@ export function CreatePythonThread(id: number, isWorker: boolean, process: IPyth
 
 export function CreatePythonModule(id: number, fileName: string): IPythonModule {
     let name = fileName;
-    if (typeof fileName === "string") {
+    if (typeof fileName === 'string') {
         try {
             name = path.basename(fileName);
-        }
-        catch (ex) {
-        }
-    }
-    else {
-        name = "";
+            // tslint:disable-next-line:no-empty
+        } catch  { }
+    } else {
+        name = '';
     }
 
     return {
@@ -74,7 +72,7 @@ export function FixupEscapedUnicodeChars(value: string): string {
 
 export function getPythonExecutable(pythonPath: string): string {
     pythonPath = untildify(pythonPath);
-    // If only 'python'
+    // If only 'python'.
     if (pythonPath === 'python' ||
         pythonPath.indexOf(path.sep) === -1 ||
         path.basename(pythonPath) === path.dirname(pythonPath)) {
@@ -84,21 +82,20 @@ export function getPythonExecutable(pythonPath: string): string {
     if (isValidPythonPath(pythonPath)) {
         return pythonPath;
     }
-    // Keep python right on top, for backwards compatibility
+    // Keep python right on top, for backwards compatibility.
     const KnownPythonExecutables = ['python', 'python4', 'python3.6', 'python3.5', 'python3', 'python2.7', 'python2'];
 
     for (let executableName of KnownPythonExecutables) {
-        // Suffix with 'python' for linux and 'osx', and 'python.exe' for 'windows'
+        // Suffix with 'python' for linux and 'osx', and 'python.exe' for 'windows'.
         if (IS_WINDOWS) {
-            executableName = executableName + '.exe';
+            executableName = `${executableName}.exe`;
             if (isValidPythonPath(path.join(pythonPath, executableName))) {
                 return path.join(pythonPath, executableName);
             }
             if (isValidPythonPath(path.join(pythonPath, 'scripts', executableName))) {
                 return path.join(pythonPath, 'scripts', executableName);
             }
-        }
-        else {
+        } else {
             if (isValidPythonPath(path.join(pythonPath, executableName))) {
                 return path.join(pythonPath, executableName);
             }
@@ -113,39 +110,36 @@ export function getPythonExecutable(pythonPath: string): string {
 
 function isValidPythonPath(pythonPath): boolean {
     try {
-        let output = child_process.execFileSync(pythonPath, ['-c', 'print(1234)'], { encoding: 'utf8' });
+        const output = child_process.execFileSync(pythonPath, ['-c', 'print(1234)'], { encoding: 'utf8' });
         return output.startsWith('1234');
-    }
-    catch (ex) {
+    } catch  {
         return false;
     }
 }
 
+type EnvVars = Object & { [key: string]: string };
 
-export function getCustomEnvVars(envVars: any, envFile: string): any {
-    let envFileVars = null;
+export function getCustomEnvVars(envVars: Object, envFile: string, mergeWithProcessEnvVars: boolean = true): EnvVars {
+    let envFileVars: EnvVars = null;
     if (typeof envFile === 'string' && envFile.length > 0 && fs.existsSync(envFile)) {
         try {
-            envFileVars = parseEnvFile(envFile);
-        }
-        catch (ex) {
+            envFileVars = parseEnvFile(envFile, mergeWithProcessEnvVars);
+        } catch (ex) {
             console.error('Failed to load env file');
             console.error(ex);
         }
     }
-    let configVars = null;
-    if (envVars && Object.keys(envVars).length > 0 && envFileVars) {
-        configVars = mergeEnvVariables(envVars, envFileVars);
-    }
-    if (envVars && Object.keys(envVars).length > 0) {
-        configVars = envVars;
-    }
-    if (envFileVars) {
-        configVars = envFileVars;
+    if (envFileVars && Object.keys(envFileVars).length > 0) {
+        if (!envVars || Object.keys(envVars).length === 0) {
+            return envFileVars;
+        } else {
+            envVars = envVars || {};
+            return mergeEnvVariables(envVars as EnvVars, envFileVars);
+        }
     }
-    if (configVars && typeof configVars === 'object' && Object.keys(configVars).length > 0) {
-        return configVars;
+    if (!envVars || Object.keys(envVars).length === 0) {
+        return null;
     }
 
-    return null;
-}
+    return mergePythonPath(envVars as EnvVars, process.env.PYTHONPATH);
+}

src/client/debugger/Common/Utils.ts

@@ -14,7 +14,7 @@ export enum DebugType {
 }
 export abstract class DebugClient extends EventEmitter {
     protected debugSession: DebugSession;
-    constructor(args: any, debugSession: DebugSession) {
+    constructor(protected args: any, debugSession: DebugSession) {
         super();
         this.debugSession = debugSession;
     }