Add telemetry for switch to Pylance acceptance (microsoft#13526)

* Fix path

* Actually fix settings

* Add news

* Add test

* Format

* Suppress 'jediEnabled' removal

* Drop survey first launch threshold

* Initial

* Add test

* PR feedback

* Rename property

* Rename constant

* Define property

* Moar rename

* casing

Author: Mikhail Arkhipov

src/client/languageServices/proposeLanguageServerBanner.ts

@@ -17,6 +17,8 @@ import {
     IPythonExtensionBanner
 } from '../common/types';
 import { Common, Pylance } from '../common/utils/localize';
+import { sendTelemetryEvent } from '../telemetry';
+import { EventName } from '../telemetry/constants';
 
 export function getPylanceExtensionUri(appEnv: IApplicationEnvironment): string {
     return `${appEnv.uriScheme}:extension/${PYLANCE_EXTENSION_ID}`;
@@ -72,14 +74,19 @@ export class ProposePylanceBanner implements IPythonExtensionBanner {
             Pylance.remindMeLater()
         );
 
+        let userAction: string;
         if (response === Pylance.tryItNow()) {
             this.appShell.openUrl(getPylanceExtensionUri(this.appEnv));
+            userAction = 'yes';
             await this.disable();
         } else if (response === Common.bannerLabelNo()) {
             await this.disable();
+            userAction = 'no';
         } else {
             this.disabledInCurrentSession = true;
+            userAction = 'later';
         }
+        sendTelemetryEvent(EventName.LANGUAGE_SERVER_TRY_PYLANCE, undefined, { userAction });
     }
 
     public async shouldShowBanner(): Promise<boolean> {

src/client/telemetry/constants.ts

@@ -93,6 +93,7 @@ export enum EventName {
     LANGUAGE_SERVER_READY = 'LANGUAGE_SERVER.READY',
     LANGUAGE_SERVER_TELEMETRY = 'LANGUAGE_SERVER.EVENT',
     LANGUAGE_SERVER_REQUEST = 'LANGUAGE_SERVER.REQUEST',
+    LANGUAGE_SERVER_TRY_PYLANCE = 'LANGUAGE_SERVER.TRY_PYLANCE',
 
     TERMINAL_CREATE = 'TERMINAL.CREATE',
     ACTIVATE_ENV_IN_CURRENT_TERMINAL = 'ACTIVATE_ENV_IN_CURRENT_TERMINAL',

src/client/telemetry/index.ts

@@ -1362,6 +1362,16 @@ export interface IEventNamePropertyMapping {
      * Telemetry sent when the client makes a request to the Node.js server
      */
     [EventName.LANGUAGE_SERVER_REQUEST]: any;
+    /**
+     * Telemetry sent on user response to 'Try Pylance' prompt.
+     */
+    [EventName.LANGUAGE_SERVER_TRY_PYLANCE]: {
+        /**
+         * User response to the prompt.
+         * @type {string}
+         */
+        userAction: string;
+    };
     /**
      * Telemetry captured for enabling reload.
      */

src/test/.vscode/settings.json

@@ -24,7 +24,6 @@
     "python.linting.banditEnabled": false,
     "python.formatting.provider": "yapf",
     "python.linting.pylintUseMinimalCheckers": false,
-    "python.pythonPath": "python",
     // Do not set this to "Microsoft", else it will result in LS being downloaded on CI
     // and that slows down tests significantly. We have other tests on CI for testing
     // downloading of LS with this setting enabled.

src/test/testing/banners/proposeNewLanguageServerBanner.unit.test.ts

@@ -2,8 +2,8 @@
 // Licensed under the MIT License.
 
 'use strict';
-
-import { expect } from 'chai';
+import { assert, expect } from 'chai';
+import * as sinon from 'sinon';
 import * as typemoq from 'typemoq';
 import { Extension } from 'vscode';
 import { LanguageServerType } from '../../../client/activation/types';
@@ -24,6 +24,8 @@ import {
     ProposeLSStateKeys,
     ProposePylanceBanner
 } from '../../../client/languageServices/proposeLanguageServerBanner';
+import * as Telemetry from '../../../client/telemetry';
+import { EventName } from '../../../client/telemetry/constants';
 
 interface IExperimentLsCombination {
     inExperiment: boolean;
@@ -46,6 +48,8 @@ suite('Propose Pylance Banner', () => {
     let appShell: typemoq.IMock<IApplicationShell>;
     let appEnv: typemoq.IMock<IApplicationEnvironment>;
     let settings: typemoq.IMock<IPythonSettings>;
+    let sendTelemetryStub: sinon.SinonStub;
+    let telemetryEvent: { eventName: EventName; properties: { userAction: string } } | undefined;
 
     const message = Pylance.proposePylanceMessage();
     const yes = Pylance.tryItNow();
@@ -59,7 +63,23 @@ suite('Propose Pylance Banner', () => {
         appShell = typemoq.Mock.ofType<IApplicationShell>();
         appEnv = typemoq.Mock.ofType<IApplicationEnvironment>();
         appEnv.setup((x) => x.uriScheme).returns(() => 'scheme');
+
+        sendTelemetryStub = sinon
+            .stub(Telemetry, 'sendTelemetryEvent')
+            .callsFake((eventName: EventName, _, properties: { userAction: string }) => {
+                telemetryEvent = {
+                    eventName,
+                    properties
+                };
+            });
+    });
+
+    teardown(() => {
+        telemetryEvent = undefined;
+        sinon.restore();
+        Telemetry._resetSharedProperties();
     });
+
     testData.forEach((t) => {
         test(`${t.inExperiment ? 'In' : 'Not in'} experiment and "python.languageServer": "${t.lsType}" should ${
             t.shouldShowBanner ? 'show' : 'not show'
@@ -109,8 +129,15 @@ suite('Propose Pylance Banner', () => {
 
         const testBanner = preparePopup(true, appShell.object, appEnv.object, config.object, true, false);
         await testBanner.showBanner();
+
         expect(testBanner.enabled).to.be.equal(false, 'Banner should be permanently disabled when user clicked No');
         appShell.verifyAll();
+
+        sinon.assert.calledOnce(sendTelemetryStub);
+        assert.deepEqual(telemetryEvent, {
+            eventName: EventName.LANGUAGE_SERVER_TRY_PYLANCE,
+            properties: { userAction: 'no' }
+        });
     });
     test('Clicking Later should disable banner in session', async () => {
         appShell
@@ -128,11 +155,20 @@ suite('Propose Pylance Banner', () => {
 
         const testBanner = preparePopup(true, appShell.object, appEnv.object, config.object, true, false);
         await testBanner.showBanner();
+
         expect(testBanner.enabled).to.be.equal(
             true,
             'Banner should not be permanently disabled when user clicked Later'
         );
         appShell.verifyAll();
+
+        sinon.assert.calledOnce(sendTelemetryStub);
+        assert.deepEqual(telemetryEvent, {
+            eventName: EventName.LANGUAGE_SERVER_TRY_PYLANCE,
+            properties: {
+                userAction: 'later'
+            }
+        });
     });
     test('Clicking Yes opens the extension marketplace entry', async () => {
         appShell
@@ -150,8 +186,17 @@ suite('Propose Pylance Banner', () => {
 
         const testBanner = preparePopup(true, appShell.object, appEnv.object, config.object, true, false);
         await testBanner.showBanner();
+
         expect(testBanner.enabled).to.be.equal(false, 'Banner should be permanently disabled after opening store URL');
         appShell.verifyAll();
+
+        sinon.assert.calledOnce(sendTelemetryStub);
+        assert.deepEqual(telemetryEvent, {
+            eventName: EventName.LANGUAGE_SERVER_TRY_PYLANCE,
+            properties: {
+                userAction: 'yes'
+            }
+        });
     });
 });