Update "Tip" notification for new users to either show the existing tip, a link to a feedback survey or nothing

news/1 Enhancements/13535.md

@@ -0,0 +1 @@
+Update "Tip" notification for new users to either show the existing tip, a link to a feedback survey or nothing.

package.json

@@ -3528,7 +3528,7 @@
         "vscode-languageclient": "^7.0.0-next.8",
         "vscode-languageserver": "^7.0.0-next.6",
         "vscode-languageserver-protocol": "^3.16.0-next.6",
-        "vscode-tas-client": "^0.0.864",
+        "vscode-tas-client": "^0.1.4",
         "vsls": "^0.3.1291",
         "winreg": "^1.2.4",
         "winston": "^3.2.1",

src/client/common/experiments/groups.ts

@@ -91,3 +91,10 @@ export enum RemoveKernelToolbarInInteractiveWindow {
 export enum TryPylance {
     experiment = 'tryPylance'
 }
+
+// Experiment for the content of the tip being displayed on first extension launch:
+// interpreter selection tip, feedback survey or nothing.
+export enum SurveyAndInterpreterTipNotification {
+    tipExperiment = 'pythonTipPromptWording',
+    surveyExperiment = 'pythonMailingListPromptWording'
+}

src/client/common/experiments/service.ts

@@ -104,6 +104,14 @@ export class ExperimentService implements IExperimentService {
         return this.experimentationService.isCachedFlightEnabled(experiment);
     }
 
+    public async getExperimentValue<T extends boolean | number | string>(experiment: string): Promise<T | undefined> {
+        if (!this.experimentationService || this._optOutFrom.includes('All') || this._optOutFrom.includes(experiment)) {
+            return;
+        }
+
+        return this.experimentationService.getTreatmentVariableAsync('vscode', experiment);
+    }
+
     private logExperiments() {
         const experiments = this.globalState.get<{ features: string[] }>(EXP_MEMENTO_KEY, { features: [] });
 

src/client/common/types.ts

@@ -637,6 +637,7 @@ export interface IExperimentsManager {
 export const IExperimentService = Symbol('IExperimentService');
 export interface IExperimentService {
     inExperiment(experimentName: string): Promise<boolean>;
+    getExperimentValue<T extends boolean | number | string>(experimentName: string): Promise<T | undefined>;
 }
 
 export type InterpreterConfigurationScope = { uri: Resource; configTarget: ConfigurationTarget };

src/client/interpreter/display/interpreterSelectionTip.ts

@@ -6,31 +6,71 @@
 import { inject, injectable } from 'inversify';
 import { IExtensionSingleActivationService } from '../../activation/types';
 import { IApplicationShell } from '../../common/application/types';
-import { IPersistentState, IPersistentStateFactory } from '../../common/types';
+import { SurveyAndInterpreterTipNotification } from '../../common/experiments/groups';
+import { IBrowserService, IExperimentService, IPersistentState, IPersistentStateFactory } from '../../common/types';
 import { swallowExceptions } from '../../common/utils/decorators';
-import { Common, Interpreters } from '../../common/utils/localize';
+import { Common } from '../../common/utils/localize';
+import { sendTelemetryEvent } from '../../telemetry';
+import { EventName } from '../../telemetry/constants';
+
+export enum NotificationType {
+    Tip,
+    Survey,
+    NoPrompt
+}
 
 @injectable()
 export class InterpreterSelectionTip implements IExtensionSingleActivationService {
     private readonly storage: IPersistentState<boolean>;
+    private notificationType: NotificationType;
+    private notificationContent: string | undefined;
+
     constructor(
         @inject(IApplicationShell) private readonly shell: IApplicationShell,
-        @inject(IPersistentStateFactory) private readonly factory: IPersistentStateFactory
+        @inject(IPersistentStateFactory) private readonly factory: IPersistentStateFactory,
+        @inject(IExperimentService) private readonly experiments: IExperimentService,
+        @inject(IBrowserService) private browserService: IBrowserService
     ) {
         this.storage = this.factory.createGlobalPersistentState('InterpreterSelectionTip', false);
+        this.notificationType = NotificationType.NoPrompt;
     }
+
     public async activate(): Promise<void> {
         if (this.storage.value) {
             return;
         }
+
+        if (await this.experiments.inExperiment(SurveyAndInterpreterTipNotification.surveyExperiment)) {
+            this.notificationType = NotificationType.Survey;
+            this.notificationContent = await this.experiments.getExperimentValue(
+                SurveyAndInterpreterTipNotification.surveyExperiment
+            );
+        } else if (await this.experiments.inExperiment(SurveyAndInterpreterTipNotification.tipExperiment)) {
+            this.notificationType = NotificationType.Tip;
+            this.notificationContent = await this.experiments.getExperimentValue(
+                SurveyAndInterpreterTipNotification.tipExperiment
+            );
+        }
+
         this.showTip().ignoreErrors();
     }
     @swallowExceptions('Failed to display tip')
     private async showTip() {
-        const selection = await this.shell.showInformationMessage(Interpreters.selectInterpreterTip(), Common.gotIt());
-        if (selection !== Common.gotIt()) {
-            return;
+        if (this.notificationType === NotificationType.Tip) {
+            await this.shell.showInformationMessage(this.notificationContent!, Common.gotIt());
+        } else if (this.notificationType === NotificationType.Survey) {
+            const selection = await this.shell.showInformationMessage(
+                this.notificationContent!,
+                Common.bannerLabelYes(),
+                Common.bannerLabelNo()
+            );
+
+            if (selection === Common.bannerLabelYes()) {
+                sendTelemetryEvent(EventName.ACTIVATION_SURVEY_PROMPT, undefined);
+                this.browserService.launch('https://aka.ms/mailingListSurvey');
+            }
         }
+
         await this.storage.updateValue(true);
     }
 }

src/client/telemetry/constants.ts

@@ -74,6 +74,8 @@ export enum EventName {
     PLAY_BUTTON_ICON_DISABLED = 'PLAY_BUTTON_ICON.DISABLED',
     PYTHON_WEB_APP_RELOAD = 'PYTHON_WEB_APP.RELOAD',
     EXTENSION_SURVEY_PROMPT = 'EXTENSION_SURVEY_PROMPT',
+    ACTIVATION_TIP_PROMPT = 'ACTIVATION_TIP_PROMPT',
+    ACTIVATION_SURVEY_PROMPT = 'ACTIVATION_SURVEY_PROMPT',
 
     PYTHON_LANGUAGE_SERVER_CURRENT_SELECTION = 'PYTHON_LANGUAGE_SERVER_CURRENT_SELECTION',
     PYTHON_LANGUAGE_SERVER_LIST_BLOB_STORE_PACKAGES = 'PYTHON_LANGUAGE_SERVER.LIST_BLOB_PACKAGES',

src/client/telemetry/index.ts

@@ -1398,6 +1398,14 @@ export interface IEventNamePropertyMapping {
          */
         selection: 'Yes' | 'Maybe later' | 'Do not show again' | undefined;
     };
+    /**
+     * Telemetry event sent when the Python interpreter tip is shown on activation for new users.
+     */
+    [EventName.ACTIVATION_TIP_PROMPT]: never | undefined;
+    /**
+     * Telemetry event sent when the feedback survey prompt is shown on activation for new users, and they click on the survey link.
+     */
+    [EventName.ACTIVATION_SURVEY_PROMPT]: never | undefined;
     /**
      * Telemetry event sent when 'Extract Method' command is invoked
      */

src/test/common/experiments/service.unit.test.ts

@@ -296,4 +296,79 @@ suite('Experimentation service', () => {
             sinon.assert.notCalled(isCachedFlightEnabledStub);
         });
     });
+
+    suite('Experiment value retrieval', () => {
+        const experiment = 'Test Experiment - experiment';
+        let getTreatmentVariableAsyncStub: sinon.SinonStub;
+
+        setup(() => {
+            getTreatmentVariableAsyncStub = sinon.stub().returns(Promise.resolve('value'));
+            sinon.stub(tasClient, 'getExperimentationService').returns({
+                getTreatmentVariableAsync: getTreatmentVariableAsyncStub
+                // tslint:disable-next-line: no-any
+            } as any);
+
+            configureApplicationEnvironment('stable', extensionVersion);
+        });
+
+        test('If the service is enabled and the opt-out array is empty,return the value from the experimentation framework for a given experiment', async () => {
+            configureSettings(true, [], []);
+
+            const experimentService = new ExperimentService(
+                instance(configurationService),
+                instance(appEnvironment),
+                globalMemento,
+                outputChannel
+            );
+            const result = await experimentService.getExperimentValue(experiment);
+
+            assert.equal(result, 'value');
+            sinon.assert.calledOnce(getTreatmentVariableAsyncStub);
+        });
+
+        test('If the experiment setting is disabled, getExperimentValue should return undefined', async () => {
+            configureSettings(false, [], []);
+
+            const experimentService = new ExperimentService(
+                instance(configurationService),
+                instance(appEnvironment),
+                globalMemento,
+                outputChannel
+            );
+            const result = await experimentService.getExperimentValue(experiment);
+
+            assert.isUndefined(result);
+            sinon.assert.notCalled(getTreatmentVariableAsyncStub);
+        });
+
+        test('If the opt-out setting contains "All", getExperimentValue should return undefined', async () => {
+            configureSettings(true, [], ['All']);
+
+            const experimentService = new ExperimentService(
+                instance(configurationService),
+                instance(appEnvironment),
+                globalMemento,
+                outputChannel
+            );
+            const result = await experimentService.getExperimentValue(experiment);
+
+            assert.isUndefined(result);
+            sinon.assert.notCalled(getTreatmentVariableAsyncStub);
+        });
+
+        test('If the opt-out setting contains the experiment name, igetExperimentValue should return undefined', async () => {
+            configureSettings(true, [], [experiment]);
+
+            const experimentService = new ExperimentService(
+                instance(configurationService),
+                instance(appEnvironment),
+                globalMemento,
+                outputChannel
+            );
+            const result = await experimentService.getExperimentValue(experiment);
+
+            assert.isUndefined(result);
+            sinon.assert.notCalled(getTreatmentVariableAsyncStub);
+        });
+    });
 });

src/test/interpreters/display/interpreterSelectionTip.unit.test.ts

@@ -6,50 +6,81 @@
 import { anything, instance, mock, verify, when } from 'ts-mockito';
 import { ApplicationShell } from '../../../client/common/application/applicationShell';
 import { IApplicationShell } from '../../../client/common/application/types';
+import { SurveyAndInterpreterTipNotification } from '../../../client/common/experiments/groups';
+import { ExperimentService } from '../../../client/common/experiments/service';
+import { BrowserService } from '../../../client/common/net/browser';
 import { PersistentState, PersistentStateFactory } from '../../../client/common/persistentState';
-import { IPersistentState } from '../../../client/common/types';
-import { Common, Interpreters } from '../../../client/common/utils/localize';
+import { IBrowserService, IExperimentService, IPersistentState } from '../../../client/common/types';
+import { Common } from '../../../client/common/utils/localize';
 import { InterpreterSelectionTip } from '../../../client/interpreter/display/interpreterSelectionTip';
 
-// tslint:disable:no-any
 suite('Interpreters - Interpreter Selection Tip', () => {
     let selectionTip: InterpreterSelectionTip;
     let appShell: IApplicationShell;
     let storage: IPersistentState<boolean>;
+    let experimentService: IExperimentService;
+    let browserService: IBrowserService;
     setup(() => {
         const factory = mock(PersistentStateFactory);
         storage = mock(PersistentState);
         appShell = mock(ApplicationShell);
+        experimentService = mock(ExperimentService);
+        browserService = mock(BrowserService);
 
         when(factory.createGlobalPersistentState('InterpreterSelectionTip', false)).thenReturn(instance(storage));
 
-        selectionTip = new InterpreterSelectionTip(instance(appShell), instance(factory));
+        selectionTip = new InterpreterSelectionTip(
+            instance(appShell),
+            instance(factory),
+            instance(experimentService),
+            instance(browserService)
+        );
     });
-    test('Do not show tip', async () => {
+    test('Do not show notification if already shown', async () => {
         when(storage.value).thenReturn(true);
 
         await selectionTip.activate();
 
         verify(appShell.showInformationMessage(anything(), anything())).never();
     });
-    test('Show tip and do not track it', async () => {
+    test('Do not show notification if in neither experiments', async () => {
         when(storage.value).thenReturn(false);
-        when(appShell.showInformationMessage(Interpreters.selectInterpreterTip(), Common.gotIt())).thenResolve();
+        when(experimentService.inExperiment(anything())).thenResolve(false);
 
         await selectionTip.activate();
 
-        verify(appShell.showInformationMessage(Interpreters.selectInterpreterTip(), Common.gotIt())).once();
-        verify(storage.updateValue(true)).never();
+        verify(appShell.showInformationMessage(anything(), anything())).never();
+        verify(storage.updateValue(true)).once();
     });
-    test('Show tip and track it', async () => {
+    test('Show tip if in tip experiment', async () => {
         when(storage.value).thenReturn(false);
-        when(appShell.showInformationMessage(Interpreters.selectInterpreterTip(), Common.gotIt())).thenResolve(
-            Common.gotIt() as any
-        );
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.tipExperiment)).thenResolve(true);
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.surveyExperiment)).thenResolve(false);
+
+        await selectionTip.activate();
+
+        verify(appShell.showInformationMessage(anything(), Common.gotIt())).once();
+        verify(storage.updateValue(true)).once();
+    });
+    test('Show survey link if in survey experiment', async () => {
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.tipExperiment)).thenResolve(false);
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.surveyExperiment)).thenResolve(true);
 
         await selectionTip.activate();
 
-        verify(appShell.showInformationMessage(Interpreters.selectInterpreterTip(), Common.gotIt())).once();
+        verify(appShell.showInformationMessage(anything(), Common.bannerLabelYes(), Common.bannerLabelNo())).once();
         verify(storage.updateValue(true)).once();
     });
+    test('Open survey link if in survey experiment and "Yes" is selected', async () => {
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.tipExperiment)).thenResolve(false);
+        when(experimentService.inExperiment(SurveyAndInterpreterTipNotification.surveyExperiment)).thenResolve(true);
+        when(appShell.showInformationMessage(anything(), Common.bannerLabelYes(), Common.bannerLabelNo())).thenResolve(
+            // tslint:disable-next-line: no-any
+            Common.bannerLabelYes() as any
+        );
+
+        await selectionTip.activate();
+
+        verify(browserService.launch(anything())).once();
+    });
 });