diff --git a/src/client/common/utils/backgroundLoop.ts b/src/client/common/utils/backgroundLoop.ts new file mode 100644 index 000000000000..f7cc78c51b73 --- /dev/null +++ b/src/client/common/utils/backgroundLoop.ts @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { createDeferred } from './async'; + +type RequestID = number; +type RunFunc = () => Promise; +type NotifyFunc = () => void; + +/** + * This helps avoid running duplicate expensive operations. + * + * The key aspect is that already running or queue requests can be + * re-used instead of creating a duplicate request. + */ +export class BackgroundRequestLooper { + private readonly opts: { + runDefault: RunFunc; + }; + + private started = false; + + private stopped = false; + + private readonly done = createDeferred(); + + private readonly loopRunning = createDeferred(); + + private waitUntilReady = createDeferred(); + + private running: RequestID | undefined; + + // For now we don't worry about a max queue size. + private readonly queue: RequestID[] = []; + + private readonly requests: Record, NotifyFunc]> = {}; + + private lastID: number | undefined; + + constructor( + opts: { + runDefault?: RunFunc | null; + } = {} + ) { + this.opts = { + runDefault: + opts.runDefault ?? + (async () => { + throw Error('no default operation provided'); + }) + }; + } + + /** + * Start the request execution loop. + * + * Currently it does not support being re-started. + */ + public start(): void { + if (this.stopped) { + throw Error('already stopped'); + } + if (this.started) { + return; + } + this.started = true; + + this.runLoop().ignoreErrors(); + } + + /** + * Stop the loop (assuming it was already started.) + * + * @returns - a promise that resolves once the loop has stopped. + */ + public stop(): Promise { + if (this.stopped) { + return this.loopRunning.promise; + } + if (!this.started) { + throw Error('not started yet'); + } + this.stopped = true; + + this.done.resolve(); + + // It is conceivable that a separate "waitUntilStopped" + // operation would be useful. If it turned out to be desirable + // then at the point we could add such a method separately. + // It would do nothing more than `await this.loopRunning`. + // Currently there is no need for a separate method since + // returning the promise here is sufficient. + return this.loopRunning.promise; + } + + /** + * Return the most recent active request, if any. + * + * If there are no pending requests then this is the currently + * running one (if one is running). + * + * @returns - the ID of the request and its completion promise; + * if there are no active requests then you get `undefined` + */ + public getLastRequest(): [RequestID, Promise] | undefined { + let reqID: RequestID; + if (this.queue.length > 0) { + reqID = this.queue[this.queue.length - 1]; + } else if (this.running !== undefined) { + reqID = this.running; + } else { + return undefined; + } + // The req cannot be undefined since every queued ID has a request. + const [, promise] = this.requests[reqID]; + if (reqID === undefined) { + // The queue must be empty. + return undefined; + } + return [reqID, promise]; + } + + /** + * Return the request that is waiting to run next, if any. + * + * The request is the next one that will be run. This implies that + * there is one already running. + * + * @returns - the ID of the request and its completion promise; + * if there are no pending requests then you get `undefined` + */ + public getNextRequest(): [RequestID, Promise] | undefined { + if (this.queue.length === 0) { + return undefined; + } + const reqID = this.queue[0]; + // The req cannot be undefined since every queued ID has a request. + const [, promise] = this.requests[reqID]!; + return [reqID, promise]; + } + + /** + * Request that a function be run. + * + * If one is already running then the new request is added to the + * end of the queue. Otherwise it is run immediately. + * + * @returns - the ID of the new request and its completion promise; + * the promise resolves once the request has completed + */ + public addRequest(run?: RunFunc): [RequestID, Promise] { + const reqID = this.getNextID(); + // This is the only method that adds requests to the queue + // and `getNextID()` keeps us from having collisions here. + // So we are guaranteed that there are no matching requests + // in the queue. + const running = createDeferred(); + this.requests[reqID] = [ + // [RunFunc, "done" promise, NotifyFunc] + run ?? this.opts.runDefault, + running.promise, + () => running.resolve() + ]; + this.queue.push(reqID); + if (this.queue.length === 1) { + // `waitUntilReady` will get replaced with a new deferred + // in the loop once the existing one gets used. + // We let the queue clear out before triggering the loop + // again. + this.waitUntilReady.resolve(); + } + return [reqID, running.promise]; + } + + /** + * This is the actual loop where the queue is managed and waiting happens. + */ + private async runLoop(): Promise { + const getWinner = () => { + const promises = [ + // These are the competing operations. + // Note that the losers keep running in the background. + this.done.promise.then(() => 0), + this.waitUntilReady.promise.then(() => 1) + ]; + return Promise.race(promises); + }; + + let winner = await getWinner(); + while (!this.done.completed) { + if (winner === 1) { + this.waitUntilReady = createDeferred(); + await this.flush(); + } else { + // This should not be reachable. + throw Error(`unsupported winner ${winner}`); + } + winner = await getWinner(); + } + this.loopRunning.resolve(); + } + + /** + * Run all pending requests, in queue order. + * + * Each request's completion promise resolves once that request + * finishes. + */ + private async flush(): Promise { + if (this.running !== undefined) { + // We must be flushing the queue already. + return; + } + // Run every request in the queue. + while (this.queue.length > 0) { + const reqID = this.queue[0]; + this.running = reqID; + // We pop the request off the queue here so it doesn't show + // up as both running and pending. + this.queue.shift(); + const [run, , notify] = this.requests[reqID]; + + await run(); + + // We leave the request until right before `notify()` + // for the sake of any calls to `getLastRequest()`. + delete this.requests[reqID]; + notify(); + } + this.running = undefined; + } + + /** + * Provide the request ID to use next. + */ + private getNextID(): RequestID { + // For now there is no way to queue up a request with + // an ID that did not originate here. So we don't need + // to worry about collisions. + if (this.lastID === undefined) { + this.lastID = 1; + } else { + this.lastID += 1; + } + return this.lastID; + } +} diff --git a/src/client/pythonEnvironments/base/envsCache.ts b/src/client/pythonEnvironments/base/envsCache.ts index f0de585b3eff..1eef95f4204e 100644 --- a/src/client/pythonEnvironments/base/envsCache.ts +++ b/src/client/pythonEnvironments/base/envsCache.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. import { cloneDeep } from 'lodash'; -import { getGlobalPersistentStore, IPersistentStore } from '../common/externalDependencies'; import { PythonEnvInfo } from './info'; import { areSameEnv } from './info/env'; @@ -13,7 +12,7 @@ export interface IEnvsCache { /** * Initialization logic to be done outside of the constructor, for example reading from persistent storage. */ - initialize(): void; + initialize(): Promise; /** * Return all environment info currently in memory for this session. @@ -30,7 +29,7 @@ export interface IEnvsCache { setAllEnvs(envs: PythonEnvInfo[]): void; /** - * If the cache has been initialized, return environmnent info objects that match a query object. + * If the cache has been initialized, return environment info objects that match a query object. * If none of the environments in the cache match the query data, return an empty array. * If the in-memory cache has not been initialized prior to calling `filterEnvs`, return `undefined`. * @@ -40,7 +39,7 @@ export interface IEnvsCache { * @return The environment info objects matching the `env` param, * or `undefined` if the in-memory cache is not initialized. */ - filterEnvs(env: PythonEnvInfo | string): PythonEnvInfo[] | undefined; + filterEnvs(query: Partial): PythonEnvInfo[] | undefined; /** * Writes the content of the in-memory cache to persistent storage. @@ -48,6 +47,11 @@ export interface IEnvsCache { flush(): Promise; } +export interface IPersistentStorage { + load(): Promise; + store(envs: PythonEnvInfo[]): Promise; +} + type CompleteEnvInfoFunction = (envInfo: PythonEnvInfo) => boolean; /** @@ -58,18 +62,23 @@ export class PythonEnvInfoCache implements IEnvsCache { private envsList: PythonEnvInfo[] | undefined; - private persistentStorage: IPersistentStore | undefined; + private persistentStorage: IPersistentStorage | undefined; - constructor(private readonly isComplete: CompleteEnvInfoFunction) {} + constructor( + private readonly isComplete: CompleteEnvInfoFunction, + private readonly getPersistentStorage?: () => IPersistentStorage, + ) {} - public initialize(): void { + public async initialize(): Promise { if (this.initialized) { return; } this.initialized = true; - this.persistentStorage = getGlobalPersistentStore('PYTHON_ENV_INFO_CACHE'); - this.envsList = this.persistentStorage?.get(); + if (this.getPersistentStorage !== undefined) { + this.persistentStorage = this.getPersistentStorage(); + this.envsList = await this.persistentStorage.load(); + } } public getAllEnvs(): PythonEnvInfo[] | undefined { @@ -80,21 +89,19 @@ export class PythonEnvInfoCache implements IEnvsCache { this.envsList = cloneDeep(envs); } - public filterEnvs(env: PythonEnvInfo | string): PythonEnvInfo[] | undefined { - const result = this.envsList?.filter((info) => areSameEnv(info, env)); - - if (result) { - return cloneDeep(result); + public filterEnvs(query: Partial): PythonEnvInfo[] | undefined { + if (this.envsList === undefined) { + return undefined; } - - return undefined; + const result = this.envsList.filter((info) => areSameEnv(info, query)); + return cloneDeep(result); } public async flush(): Promise { const completeEnvs = this.envsList?.filter(this.isComplete); if (completeEnvs?.length) { - await this.persistentStorage?.set(completeEnvs); + await this.persistentStorage?.store(completeEnvs); } } } diff --git a/src/client/pythonEnvironments/base/locators/composite/cachingLocator.ts b/src/client/pythonEnvironments/base/locators/composite/cachingLocator.ts new file mode 100644 index 000000000000..511dc1b8d4c2 --- /dev/null +++ b/src/client/pythonEnvironments/base/locators/composite/cachingLocator.ts @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Event } from 'vscode'; +import '../../../../common/extensions'; +import { createDeferred } from '../../../../common/utils/async'; +import { BackgroundRequestLooper } from '../../../../common/utils/backgroundLoop'; +import { logWarning } from '../../../../logging'; +import { IEnvsCache } from '../../envsCache'; +import { PythonEnvInfo } from '../../info'; +import { getMinimalPartialInfo } from '../../info/env'; +import { + ILocator, + IPythonEnvsIterator, + PythonLocatorQuery, +} from '../../locator'; +import { getEnvs, getQueryFilter } from '../../locatorUtils'; +import { PythonEnvsChangedEvent, PythonEnvsWatcher } from '../../watcher'; +import { pickBestEnv } from './reducingLocator'; + +/** + * A locator that stores the known environments in the given cache. + */ +export class CachingLocator implements ILocator { + public readonly onChanged: Event; + + private readonly watcher = new PythonEnvsWatcher(); + + private readonly initializing = createDeferred(); + + private initialized = false; + + private looper: BackgroundRequestLooper; + + constructor( + private readonly cache: IEnvsCache, + private readonly locator: ILocator, + ) { + this.onChanged = this.watcher.onChanged; + this.looper = new BackgroundRequestLooper({ + runDefault: null, + }); + } + + /** + * Prepare the locator for use. + * + * This must be called before using the locator. It is distinct + * from the constructor to avoid the problems that come from doing + * any serious work in constructors. It also allows initialization + * to be asynchronous. + */ + public async initialize(): Promise { + if (this.initialized) { + return; + } + + await this.cache.initialize(); + this.looper.start(); + + this.locator.onChanged((event) => this.ensureCurrentRefresh(event)); + + // Do the initial refresh. + const envs = this.cache.getAllEnvs(); + if (envs !== undefined) { + this.initializing.resolve(); + await this.ensureRecentRefresh(); + } else { + // There is nothing in the cache, so we must wait for the + // initial refresh to finish before allowing iteration. + await this.ensureRecentRefresh(); + this.initializing.resolve(); + } + } + + public dispose(): void { + const waitUntilStopped = this.looper.stop(); + waitUntilStopped.ignoreErrors(); + } + + public iterEnvs(query?: PythonLocatorQuery): IPythonEnvsIterator { + // We assume that `getAllEnvs()` is cheap enough that calling + // it again in `iterFromCache()` is not a problem. + if (this.cache.getAllEnvs() === undefined) { + return this.iterFromWrappedLocator(query); + } + return this.iterFromCache(query); + } + + public async resolveEnv(env: string | PythonEnvInfo): Promise { + // If necessary we could be more aggressive about invalidating + // the cached value. + const query = getMinimalPartialInfo(env); + if (query === undefined) { + return undefined; + } + const candidates = this.cache.filterEnvs(query); + if (candidates === undefined) { + return undefined; + } + if (candidates.length > 0) { + return pickBestEnv(candidates); + } + // Fall back to the underlying locator. + const resolved = await this.locator.resolveEnv(env); + if (resolved !== undefined) { + const envs = this.cache.getAllEnvs(); + if (envs !== undefined) { + envs.push(resolved); + await this.updateCache(envs); + } + } + return resolved; + } + + /** + * A generator that yields the envs provided by the wrapped locator. + * + * Contrast this with `iterFromCache()` that yields only from the cache. + */ + private async* iterFromWrappedLocator(query?: PythonLocatorQuery): IPythonEnvsIterator { + // For now we wait for the initial refresh to finish. If that + // turns out to be a problem then we can do something more + // clever here. + await this.initializing.promise; + const iterator = this.iterFromCache(query); + let res = await iterator.next(); + while (!res.done) { + yield res.value; + res = await iterator.next(); + } + } + + /** + * A generator that yields the envs found in the cache. + * + * Contrast this with `iterFromWrappedLocator()`. + */ + private async* iterFromCache(query?: PythonLocatorQuery): IPythonEnvsIterator { + const envs = this.cache.getAllEnvs(); + if (envs === undefined) { + logWarning('envs cache unexpectedly not initialized'); + return; + } + // We trust `this.locator.onChanged` to be reliable. + // So there is no need to check if anything is stale + // at this point. + if (query !== undefined) { + const filter = getQueryFilter(query); + yield* envs.filter(filter); + } else { + yield* envs; + } + } + + /** + * Maybe trigger a refresh of the cache from the wrapped locator. + * + * If a refresh isn't already running then we request a refresh and + * wait for it to finish. Otherwise we do not make a new request, + * but instead only wait for the last requested refresh to complete. + */ + private ensureRecentRefresh(): Promise { + // Re-use the last req in the queue if possible. + const last = this.looper.getLastRequest(); + if (last !== undefined) { + const [, promise] = last; + return promise; + } + // The queue is empty so add a new request. + return this.addRefreshRequest(); + } + + /** + * Maybe trigger a refresh of the cache from the wrapped locator. + * + * Make sure that a completely new refresh will be started soon and + * wait for it to finish. If a refresh isn't already running then + * we start one and wait for it to finish. If one is already + * running then we make sure a new one is requested to start after + * that and wait for it to finish. That means if one is already + * waiting in the queue then we wait for that one instead of making + * a new request. + */ + private ensureCurrentRefresh(event?: PythonEnvsChangedEvent): void { + const req = this.looper.getNextRequest(); + if (req === undefined) { + // There isn't already a pending request (due to an + // onChanged event), so we add one. + this.addRefreshRequest(event) + .ignoreErrors(); + } + // Otherwise let the pending request take care of it. + } + + /** + * Queue up a new request to refresh the cache from the wrapped locator. + * + * Once the request is added, that refresh will run no matter what + * at some future point (possibly immediately). It does not matter + * if another refresh is already running. You probably want to use + * `ensureRecentRefresh()` or * `ensureCurrentRefresh()` instead, + * to avoid unnecessary refreshes. + */ + private addRefreshRequest( + event?: PythonEnvsChangedEvent, + ): Promise { + const [, waitUntilDone] = this.looper.addRequest(async () => { + const iterator = this.locator.iterEnvs(); + const envs = await getEnvs(iterator); + await this.updateCache(envs, event); + }); + return waitUntilDone; + } + + /** + * Set the cache to the given envs, flush, and emit an onChanged event. + */ + private async updateCache( + envs: PythonEnvInfo[], + event?: PythonEnvsChangedEvent, + ): Promise { + // If necessary, we could skip if there are no changes. + this.cache.setAllEnvs(envs); + await this.cache.flush(); + this.watcher.fire(event || {}); // Emit an "onChanged" event. + } +} diff --git a/src/client/pythonEnvironments/base/locators/composite/reducingLocator.ts b/src/client/pythonEnvironments/base/locators/composite/reducingLocator.ts new file mode 100644 index 000000000000..8dc71c28d699 --- /dev/null +++ b/src/client/pythonEnvironments/base/locators/composite/reducingLocator.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { PythonEnvInfo } from '../../info'; + +/** + * Determine which of the given envs should be used. + * + * The candidates must be equivalent in some way. + */ +export function pickBestEnv(candidates: PythonEnvInfo[]): PythonEnvInfo { + // For the moment we take a naive approach. + return candidates[0]; +} diff --git a/src/client/pythonEnvironments/index.ts b/src/client/pythonEnvironments/index.ts index 34089990d5fc..600c9aafd6c0 100644 --- a/src/client/pythonEnvironments/index.ts +++ b/src/client/pythonEnvironments/index.ts @@ -6,14 +6,16 @@ import { IServiceContainer, IServiceManager } from '../ioc/types'; import { PythonEnvInfoCache } from './base/envsCache'; import { PythonEnvInfo } from './base/info'; import { ILocator, IPythonEnvsIterator, PythonLocatorQuery } from './base/locator'; +import { CachingLocator } from './base/locators/composite/cachingLocator'; import { PythonEnvsChangedEvent } from './base/watcher'; +import { getGlobalPersistentStore } from './common/externalDependencies'; import { ExtensionLocators, WorkspaceLocators } from './discovery/locators'; import { registerForIOC } from './legacyIOC'; /** * Activate the Python environments component (during extension activation).' */ -export function activate(serviceManager: IServiceManager, serviceContainer: IServiceContainer) { +export function activate(serviceManager: IServiceManager, serviceContainer: IServiceContainer): void { const [api, activateAPI] = createAPI(); registerForIOC(serviceManager, serviceContainer, api); activateAPI(); @@ -27,7 +29,7 @@ export function activate(serviceManager: IServiceManager, serviceContainer: ISer export class PythonEnvironments implements ILocator { constructor( // These are the sub-components the full component is composed of: - private readonly locators: ILocator + private readonly locators: ILocator, ) {} public get onChanged(): vscode.Event { @@ -52,14 +54,25 @@ export function createAPI(): [PythonEnvironments, () => void] { const [locators, activateLocators] = initLocators(); // Update this to pass in an actual function that checks for env info completeness. - const envsCache = new PythonEnvInfoCache(() => true); + const envsCache = new PythonEnvInfoCache( + () => true, // "isComplete" + () => { + const storage = getGlobalPersistentStore('PYTHON_ENV_INFO_CACHE'); + return { + load: async () => storage.get(), + store: async (e) => storage.set(e), + }; + }, + ); + const cachingLocator = new CachingLocator(envsCache, locators); return [ - new PythonEnvironments(locators), + new PythonEnvironments(cachingLocator), () => { activateLocators(); + envsCache.initialize().ignoreErrors(); + cachingLocator.initialize().ignoreErrors(); // Any other activation needed for the API will go here later. - envsCache.initialize(); }, ]; } @@ -81,7 +94,7 @@ function initLocators(): [ExtensionLocators, () => void] { () => { // Any non-workspace locator activation goes here. workspaceLocators.activate(getWorkspaceFolders()); - } + }, ]; } @@ -100,6 +113,6 @@ function getWorkspaceFolders() { return { roots: folders ? folders.map((f) => f.uri) : [], onAdded: rootAdded.event, - onRemoved: rootRemoved.event + onRemoved: rootRemoved.event, }; } diff --git a/src/test/pythonEnvironments/base/common.ts b/src/test/pythonEnvironments/base/common.ts index 62077e46aa20..1e78ed20f59d 100644 --- a/src/test/pythonEnvironments/base/common.ts +++ b/src/test/pythonEnvironments/base/common.ts @@ -50,7 +50,7 @@ export class SimpleLocator extends Locator { private deferred = createDeferred(); constructor( private envs: PythonEnvInfo[], - private callbacks?: { + public callbacks: { resolve?: null | ((env: PythonEnvInfo) => Promise); before?: Promise; after?: Promise; @@ -58,7 +58,7 @@ export class SimpleLocator extends Locator { beforeEach?(e: PythonEnvInfo): Promise; afterEach?(e: PythonEnvInfo): Promise; onQuery?(query: PythonLocatorQuery | undefined, envs: PythonEnvInfo[]): Promise; - } + } = {}, ) { super(); } @@ -73,13 +73,13 @@ export class SimpleLocator extends Locator { const callbacks = this.callbacks; let envs = this.envs; const iterator: IPythonEnvsIterator = async function*() { - if (callbacks?.onQuery !== undefined) { + if (callbacks.onQuery !== undefined) { envs = await callbacks.onQuery(query, envs); } - if (callbacks?.before !== undefined) { + if (callbacks.before !== undefined) { await callbacks.before; } - if (callbacks?.beforeEach !== undefined) { + if (callbacks.beforeEach !== undefined) { // The results will likely come in a different order. const mapped = mapToIterator(envs, async (env) => { await callbacks.beforeEach!(env); @@ -87,31 +87,31 @@ export class SimpleLocator extends Locator { }); for await (const env of iterable(mapped)) { yield env; - if (callbacks?.afterEach !== undefined) { + if (callbacks.afterEach !== undefined) { await callbacks.afterEach(env); } } } else { for (const env of envs) { yield env; - if (callbacks?.afterEach !== undefined) { + if (callbacks.afterEach !== undefined) { await callbacks.afterEach(env); } } } - if (callbacks?.after!== undefined) { + if (callbacks.after!== undefined) { await callbacks.after; } deferred.resolve(); }(); - iterator.onUpdated = this.callbacks?.onUpdated; + iterator.onUpdated = this.callbacks.onUpdated; return iterator; } public async resolveEnv(env: string | PythonEnvInfo): Promise { const envInfo: PythonEnvInfo = typeof env === 'string' ? createLocatedEnv('', '', undefined, env) : env; - if (this.callbacks?.resolve === undefined) { + if (this.callbacks.resolve === undefined) { return envInfo; - } else if (this.callbacks?.resolve === null) { + } else if (this.callbacks.resolve === null) { return undefined; } else { return this.callbacks.resolve(envInfo); diff --git a/src/test/pythonEnvironments/base/envsCache.unit.test.ts b/src/test/pythonEnvironments/base/envsCache.unit.test.ts index 530711063910..b0dcb252629d 100644 --- a/src/test/pythonEnvironments/base/envsCache.unit.test.ts +++ b/src/test/pythonEnvironments/base/envsCache.unit.test.ts @@ -35,33 +35,46 @@ suite('Environment Info cache', () => { }); }); + function getGlobalPersistentStore() { + // It may look like we are making this call directly, but note + // that in `setup()` we have already stubbed the function out. + // We take this approach so the tests more closely match how + // `PythonEnvInfoCache` will actually be used in the VS Code + // extension. + const store = externalDependencies.getGlobalPersistentStore('PYTHON_ENV_INFO_CACHE'); + return { + load: async () => store.get(), + store: (envs: PythonEnvInfo[]) => store.set(envs), + }; + } + teardown(() => { getGlobalPersistentStoreStub.restore(); updatedValues = undefined; }); - test('`initialize` reads from persistent storage', () => { - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + test('`initialize` reads from persistent storage', async () => { + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); - envsCache.initialize(); + await envsCache.initialize(); assert.ok(getGlobalPersistentStoreStub.calledOnce); }); - test('The in-memory env info array is undefined if there is no value in persistent storage when initializing the cache', () => { - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + test('The in-memory env info array is undefined if there is no value in persistent storage when initializing the cache', async () => { + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); getGlobalPersistentStoreStub.returns({ get() { return undefined; } }); - envsCache.initialize(); + await envsCache.initialize(); const result = envsCache.getAllEnvs(); assert.strictEqual(result, undefined); }); - test('`getAllEnvs` should return a deep copy of the environments currently in memory', () => { - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + test('`getAllEnvs` should return a deep copy of the environments currently in memory', async () => { + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); - envsCache.initialize(); + await envsCache.initialize(); const envs = envsCache.getAllEnvs()!; envs[0].name = 'some-other-name'; @@ -70,7 +83,7 @@ suite('Environment Info cache', () => { }); test('`getAllEnvs` should return undefined if nothing has been set', () => { - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); const envs = envsCache.getAllEnvs(); @@ -78,7 +91,7 @@ suite('Environment Info cache', () => { }); test('`setAllEnvs` should clone the environment info array passed as a parameter', () => { - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); envsCache.setAllEnvs(envInfoArray); const envs = envsCache.getAllEnvs(); @@ -87,11 +100,11 @@ suite('Environment Info cache', () => { assert.strictEqual(envs === envInfoArray, false); }); - test('`filterEnvs` should return environments that match its argument using areSameEnvironmnet', () => { + test('`filterEnvs` should return environments that match its argument using areSameEnvironmnet', async () => { const env:PythonEnvInfo = { executable: { filename: 'my-venv-env' } } as unknown as PythonEnvInfo; - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); - envsCache.initialize(); + await envsCache.initialize(); const result = envsCache.filterEnvs(env); @@ -105,7 +118,7 @@ suite('Environment Info cache', () => { kind: PythonEnvKind.System, executable: { filename: 'my-system-env' }, } as unknown as PythonEnvInfo; const env:PythonEnvInfo = { executable: { filename: 'my-system-env' } } as unknown as PythonEnvInfo; - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); envsCache.setAllEnvs([...envInfoArray, envToFind]); @@ -115,11 +128,11 @@ suite('Environment Info cache', () => { assert.notDeepStrictEqual(result[0], envToFind); }); - test('`filterEnvs` should return an empty array if no environment matches the properties of its argument', () => { + test('`filterEnvs` should return an empty array if no environment matches the properties of its argument', async () => { const env:PythonEnvInfo = { executable: { filename: 'my-nonexistent-env' } } as unknown as PythonEnvInfo; - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); - envsCache.initialize(); + await envsCache.initialize(); const result = envsCache.filterEnvs(env); @@ -128,7 +141,7 @@ suite('Environment Info cache', () => { test('`filterEnvs` should return undefined if the cache hasn\'t been initialized', () => { const env:PythonEnvInfo = { executable: { filename: 'my-nonexistent-env' } } as unknown as PythonEnvInfo; - const envsCache = new PythonEnvInfoCache(allEnvsComplete); + const envsCache = new PythonEnvInfoCache(allEnvsComplete, getGlobalPersistentStore); const result = envsCache.filterEnvs(env); @@ -147,9 +160,12 @@ suite('Environment Info cache', () => { const expected = [ otherEnv, ]; - const envsCache = new PythonEnvInfoCache((env) => env.defaultDisplayName !== undefined); + const envsCache = new PythonEnvInfoCache( + (env) => env.defaultDisplayName !== undefined, + getGlobalPersistentStore, + ); - envsCache.initialize(); + await envsCache.initialize(); envsCache.setAllEnvs(updatedEnvInfoArray); await envsCache.flush(); @@ -157,7 +173,10 @@ suite('Environment Info cache', () => { }); test('`flush` should not write to persistent storage if there are no environment info objects in-memory', async () => { - const envsCache = new PythonEnvInfoCache((env) => env.kind === PythonEnvKind.MacDefault); + const envsCache = new PythonEnvInfoCache( + (env) => env.kind === PythonEnvKind.MacDefault, + getGlobalPersistentStore, + ); await envsCache.flush(); @@ -165,9 +184,12 @@ suite('Environment Info cache', () => { }); test('`flush` should not write to persistent storage if there are no complete environment info objects', async () => { - const envsCache = new PythonEnvInfoCache((env) => env.kind === PythonEnvKind.MacDefault); + const envsCache = new PythonEnvInfoCache( + (env) => env.kind === PythonEnvKind.MacDefault, + getGlobalPersistentStore, + ); - envsCache.initialize(); + await envsCache.initialize(); await envsCache.flush(); assert.strictEqual(updatedValues, undefined); diff --git a/src/test/pythonEnvironments/base/locators/composite/cachingLocator.unit.test.ts b/src/test/pythonEnvironments/base/locators/composite/cachingLocator.unit.test.ts new file mode 100644 index 000000000000..e4051b390dca --- /dev/null +++ b/src/test/pythonEnvironments/base/locators/composite/cachingLocator.unit.test.ts @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as path from 'path'; +import { Uri } from 'vscode'; +import { createDeferred } from '../../../../../client/common/utils/async'; +import { PythonEnvInfoCache } from '../../../../../client/pythonEnvironments/base/envsCache'; +import { PythonEnvInfo, PythonEnvKind } from '../../../../../client/pythonEnvironments/base/info'; +import { CachingLocator } from '../../../../../client/pythonEnvironments/base/locators/composite/cachingLocator'; +import { getEnvs } from '../../../../../client/pythonEnvironments/base/locatorUtils'; +import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher'; +import { + createLocatedEnv, + createNamedEnv, + SimpleLocator, +} from '../../common'; + +const env1 = createNamedEnv('env1', '2.7.11', PythonEnvKind.System, '/usr/bin/python'); +const env2 = createNamedEnv('env2', '3.8.1', PythonEnvKind.System, '/usr/bin/python3'); +const env3 = createLocatedEnv('/a/b/c/env5', '3.8.1', PythonEnvKind.Pipenv); +env3.searchLocation = Uri.file(path.normalize('/a/b/c')); +const env4 = createLocatedEnv('/x/y/z/env3', '2.7.11', PythonEnvKind.Venv); +env4.searchLocation = Uri.file(path.normalize('/x/y/z')); +const env5 = createLocatedEnv('/x/y/z/env4', '3.8.1', PythonEnvKind.Venv); +env5.searchLocation = Uri.file(path.normalize('/x/y/z')); +const envs = [env1, env2, env3, env4, env5]; + +class FakeCache extends PythonEnvInfoCache { + constructor( + load: () => Promise, + store: (e: PythonEnvInfo[]) => Promise, + isComplete: (e: PythonEnvInfo) => boolean = () => true, + ) { + super(isComplete, () => ({ load, store })); + } +} + +async function getInitializedLocator(initialEnvs: PythonEnvInfo[]): Promise<[SimpleLocator, CachingLocator]> { + const cache = new FakeCache( + () => Promise.resolve(undefined), + () => Promise.resolve(undefined), + ); + const subLocator = new SimpleLocator(initialEnvs, { + resolve: null, + }); + const locator = new CachingLocator(cache, subLocator); + await locator.initialize(); + return [subLocator, locator]; +} + +suite('Python envs locator - CachingLocator', () => { + suite('initialize', () => { + test('cache initialized', async () => { + const loadDeferred = createDeferred(); + const storeDeferred = createDeferred(); + let storedEnvs: PythonEnvInfo[] | undefined; + const cache = new FakeCache( + () => { + const promise = Promise.resolve([env1]); + promise.then(() => loadDeferred.resolve()).ignoreErrors(); + return promise; + }, + async (e) => { + storedEnvs = e; + storeDeferred.resolve(); + }, + ); + const subDeferred = createDeferred(); + const subLocator = new SimpleLocator([env2], { + before: (async () => { + if (subDeferred.completed) { + throw Error('called more than once!'); + } + await subDeferred.promise; + })(), + }); + const locator = new CachingLocator(cache, subLocator); + + locator.initialize().ignoreErrors(); // in the background + await loadDeferred.promise; // This lets the load finish. + const resultBefore = await getEnvs(locator.iterEnvs()); + subDeferred.resolve(); // This lets the refresh continue. + await storeDeferred.promise; // This lets the refresh finish. + const resultAfter = await getEnvs(locator.iterEnvs()); + + assert.deepEqual(storedEnvs, [env2]); + assert.deepEqual(resultBefore, [env1]); + assert.deepEqual(resultAfter, [env2]); + }); + }); + + suite('onChanged', () => { + test('emitted after initial refresh', async () => { + const expected: PythonEnvsChangedEvent = {}; + const cache = new FakeCache( + () => Promise.resolve(undefined), + () => Promise.resolve(undefined), + ); + const subLocator = new SimpleLocator([env2]); + const locator = new CachingLocator(cache, subLocator); + + let changeEvent: PythonEnvsChangedEvent | undefined; + locator.onChanged((e) => { changeEvent = e; }); + await locator.initialize(); + + assert.deepEqual(changeEvent, expected); + }); + + test('propagated', async () => { + const expected: PythonEnvsChangedEvent = {}; + const [subLocator, locator] = await getInitializedLocator([env2]); + let changeEvent: PythonEnvsChangedEvent | undefined; + const eventDeferred = createDeferred(); + + locator.onChanged((e) => { + changeEvent = e; + eventDeferred.resolve(); + }); + subLocator.fire({}); + await eventDeferred.promise; + + assert.deepEqual(changeEvent, expected); + }); + }); + + suite('iterEnvs()', () => { + test('no query', async () => { + const expected = envs; + const [, locator] = await getInitializedLocator(envs); + + const iterator = locator.iterEnvs(); + const discovered = await getEnvs(iterator); + + assert.deepEqual(discovered, expected); + }); + + test('filter kinds', async () => { + const expected = [env1, env2, env4, env5]; + const [, locator] = await getInitializedLocator(envs); + const query = { + kinds: [ + PythonEnvKind.Venv, + PythonEnvKind.System, + ], + }; + + const iterator = locator.iterEnvs(query); + const discovered = await getEnvs(iterator); + + assert.deepEqual(discovered, expected); + }); + + test('filter locations', async () => { + const expected = [env4, env5]; + const query = { + searchLocations: { + roots: [Uri.file(path.normalize('/x/y/z'))], + }, + }; + const [, locator] = await getInitializedLocator(envs); + + const iterator = locator.iterEnvs(query); + const discovered = await getEnvs(iterator); + + assert.deepEqual(discovered, expected); + }); + + test('cache empty', async () => { + const [, locator] = await getInitializedLocator([]); + + const iterator = locator.iterEnvs(); + const discovered = await getEnvs(iterator); + + assert.deepEqual(discovered, []); + }); + }); + + suite('resolveEnv()', () => { + test('full match in cache', async () => { + const expected = env5; + const [, locator] = await getInitializedLocator(envs); + + const resolved = await locator.resolveEnv(env5); + + assert.deepEqual(resolved, expected); + }); + + test('executable match in cache', async () => { + const expected = env5; + const [, locator] = await getInitializedLocator(envs); + + const resolved = await locator.resolveEnv(env5.executable.filename); + + assert.deepEqual(resolved, expected); + }); + + test('not in cache but found downstream', async () => { + const expected = env5; + const [subLocator, locator] = await getInitializedLocator([]); + subLocator.callbacks.resolve = () => Promise.resolve(env5); + + const iterator1 = locator.iterEnvs(); + const discoveredBefore = await getEnvs(iterator1); + const resolved = await locator.resolveEnv(env5); + const iterator2 = locator.iterEnvs(); + const discoveredAfter = await getEnvs(iterator2); + + assert.deepEqual(resolved, expected); + assert.deepEqual(discoveredBefore, []); + assert.deepEqual(discoveredAfter, [env5]); + }); + + test('not in cache nor downstream', async () => { + const [, locator] = await getInitializedLocator([]); + + const resolved = await locator.resolveEnv(env5); + + assert.equal(resolved, undefined); + }); + }); +});