Identify conda and windows store environments from given interpreter path (#13589)

* Initial commit

* Implement Windows store identifier

* Fix test errors.

* Add more notes on windows store identifier.

* minor tweaks.

* Address comments.

* Clean up

* Missed suggestions.

Author: karthiknadig

src/client/common/utils/platform.ts

@@ -3,6 +3,8 @@
 
 'use strict';
 
+import { EnvironmentVariables } from '../variables/types';
+
 export enum Architecture {
     Unknown = 1,
     x86 = 2,
@@ -27,3 +29,19 @@ export function getOSType(platform: string = process.platform): OSType {
         return OSType.Unknown;
     }
 }
+
+export function getEnvironmentVariable(key: string): string | undefined {
+    // tslint:disable-next-line: no-any
+    return ((process.env as any) as EnvironmentVariables)[key];
+}
+
+export function getPathEnvironmentVariable(): string | undefined {
+    return getEnvironmentVariable('Path') || getEnvironmentVariable('PATH');
+}
+
+export function getUserHomeDir(): string | undefined {
+    if (getOSType() === OSType.Windows) {
+        return getEnvironmentVariable('USERPROFILE');
+    }
+    return getEnvironmentVariable('HOME') || getEnvironmentVariable('HOMEPATH');
+}

src/client/pythonEnvironments/common/environmentIdentifier.ts

@@ -0,0 +1,160 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import * as path from 'path';
+import { traceWarning } from '../../common/logger';
+import { createDeferred } from '../../common/utils/async';
+import { getEnvironmentVariable } from '../../common/utils/platform';
+import { EnvironmentType } from '../info';
+
+function pathExists(absPath: string): Promise<boolean> {
+    const deferred = createDeferred<boolean>();
+    fsapi.exists(absPath, (result) => {
+        deferred.resolve(result);
+    });
+    return deferred.promise;
+}
+
+/**
+ * Checks if the given interpreter path belongs to a conda environment. Using
+ * known folder layout, and presence of 'conda-meta' directory.
+ * @param {string} interpreterPath: Absolute path to any python interpreter.
+ *
+ * Remarks: This is what we will use to begin with. Another approach we can take
+ * here is to parse ~/.conda/environments.txt. This file will have list of conda
+ * environments. We can compare the interpreter path against the paths in that file.
+ * We don't want to rely on this file because it is an implementation detail of
+ * conda. If it turns out that the layout based identification is not sufficient
+ * that is the next alternative that is cheap.
+ *
+ * sample content of the ~/.conda/environments.txt:
+ * C:\envs\\myenv
+ * C:\ProgramData\Miniconda3
+ *
+ * Yet another approach is to use `conda env list --json` and compare the returned env
+ * list to see if the given interpreter path belongs to any of the returned environments.
+ * This approach is heavy, and involves running a binary. For now we decided not to
+ * take this approach, since it does not look like we need it.
+ *
+ * sample output from `conda env list --json`:
+ * conda env list --json
+ * {
+ *   "envs": [
+ *     "C:\\envs\\myenv",
+ *     "C:\\ProgramData\\Miniconda3"
+ *   ]
+ * }
+ */
+async function isCondaEnvironment(interpreterPath: string): Promise<boolean> {
+    const condaMetaDir = 'conda-meta';
+
+    // Check if the conda-meta directory is in the same directory as the interpreter.
+    // This layout is common in Windows.
+    // env
+    // |__ conda-meta  <--- check if this directory exists
+    // |__ python.exe  <--- interpreterPath
+    const condaEnvDir1 = path.join(path.dirname(interpreterPath), condaMetaDir);
+
+    // Check if the conda-meta directory is in the parent directory relative to the interpreter.
+    // This layout is common on linux/Mac.
+    // env
+    // |__ conda-meta  <--- check if this directory exists
+    // |__ bin
+    //     |__ python  <--- interpreterPath
+    const condaEnvDir2 = path.join(path.dirname(path.dirname(interpreterPath)), condaMetaDir);
+
+    return [await pathExists(condaEnvDir1), await pathExists(condaEnvDir2)].includes(true);
+}
+
+/**
+ * Checks if the given interpreter belongs to Windows Store Python environment.
+ * @param interpreterPath: Absolute path to any python interpreter.
+ *
+ * Remarks:
+ * 1. Checking if the path includes 'Microsoft\WindowsApps`, `Program Files\WindowsApps`, is
+ * NOT enough. In WSL, /mnt/c/users/user/AppData/Local/Microsoft/WindowsApps is available as a search
+ * path. It is possible to get a false positive for that path. So the comparison should check if the
+ * absolute path to 'WindowsApps' directory is present in the given interpreter path. The WSL path to
+ * 'WindowsApps' is not a valid path to access, Windows Store Python.
+ *
+ * 2. 'startsWith' comparison may not be right, user can provide '\\?\C:\users\' style long paths in windows.
+ *
+ * 3. A limitation of the checks here is that they don't handle 8.3 style windows paths.
+ * For example,
+ *     C:\Users\USER\AppData\Local\MICROS~1\WINDOW~1\PYTHON~2.EXE
+ * is the shortened form of
+ *     C:\Users\USER\AppData\Local\Microsoft\WindowsApps\python3.7.exe
+ *
+ * The correct way to compare these would be to always convert given paths to long path (or to short path).
+ * For either approach to work correctly you need actual file to exist, and accessible from the user's
+ * account.
+ *
+ * To convert to short path without using N-API in node would be to use this command. This is very expensive:
+ * > cmd /c for %A in ("C:\Users\USER\AppData\Local\Microsoft\WindowsApps\python3.7.exe") do @echo %~sA
+ * The above command will print out this:
+ * C:\Users\USER\AppData\Local\MICROS~1\WINDOW~1\PYTHON~2.EXE
+ *
+ * If we go down the N-API route, use node-ffi and either call GetShortPathNameW or GetLongPathNameW from,
+ * Kernel32 to convert between the two path variants.
+ *
+ */
+async function isWindowsStoreEnvironment(interpreterPath: string): Promise<boolean> {
+    const pythonPathToCompare = path.normalize(interpreterPath).toUpperCase();
+    const localAppDataStorePath = path
+        .join(getEnvironmentVariable('LOCALAPPDATA') || '', 'Microsoft', 'WindowsApps')
+        .normalize()
+        .toUpperCase();
+    if (pythonPathToCompare.includes(localAppDataStorePath)) {
+        return true;
+    }
+
+    // Program Files store path is a forbidden path. Only admins and system has access this path.
+    // We should never have to look at this path or even execute python from this path.
+    const programFilesStorePath = path
+        .join(getEnvironmentVariable('ProgramFiles') || 'Program Files', 'WindowsApps')
+        .normalize()
+        .toUpperCase();
+    if (pythonPathToCompare.includes(programFilesStorePath)) {
+        traceWarning('isWindowsStoreEnvironment called with Program Files store path.');
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Returns environment type.
+ * @param {string} interpreterPath : Absolute path to the python interpreter binary.
+ * @returns {EnvironmentType}
+ *
+ * Remarks: This is the order of detection based on how the various distributions and tools
+ * configure the environment, and the fall back for identification.
+ * Top level we have the following environment types, since they leave a unique signature
+ * in the environment or * use a unique path for the environments they create.
+ *  1. Conda
+ *  2. Windows Store
+ *  3. PipEnv
+ *  4. Pyenv
+ *  5. Poetry
+ *
+ * Next level we have the following virtual environment tools. The are here because they
+ * are consumed by the tools above, and can also be used independently.
+ *  1. venv
+ *  2. virtualenvwrapper
+ *  3. virtualenv
+ *
+ * Last category is globally installed python, or system python.
+ */
+export async function identifyEnvironment(interpreterPath: string): Promise<EnvironmentType> {
+    if (await isCondaEnvironment(interpreterPath)) {
+        return EnvironmentType.Conda;
+    }
+
+    if (await isWindowsStoreEnvironment(interpreterPath)) {
+        return EnvironmentType.WindowsStore;
+    }
+
+    // additional identifiers go here
+
+    return EnvironmentType.Unknown;
+}

src/client/pythonEnvironments/discovery/locators/helpers.ts

@@ -21,7 +21,7 @@ export async function lookForInterpretersInDirectory(pathToCheck: string, _: IFi
             .map((filename) => path.join(pathToCheck, filename))
             .filter((fileName) => CheckPythonInterpreterRegEx.test(path.basename(fileName)));
     } catch (err) {
-        traceError('Python Extension (lookForInterpretersInDirectory.fs.listdir):', err);
+        traceError('Python Extension (lookForInterpretersInDirectory.fs.readdir):', err);
         return [] as string[];
     }
 }

src/test/pythonEnvironments/common/environmentIdentifier.unit.test.ts

@@ -0,0 +1,110 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as assert from 'assert';
+import * as path from 'path';
+import * as sinon from 'sinon';
+import * as platformApis from '../../../client/common/utils/platform';
+import { identifyEnvironment } from '../../../client/pythonEnvironments/common/environmentIdentifier';
+import { EnvironmentType } from '../../../client/pythonEnvironments/info';
+
+suite('Environment Identifier', () => {
+    const testLayoutsRoot = path.join(
+        __dirname,
+        '..',
+        '..',
+        '..',
+        '..',
+        'src',
+        'test',
+        'pythonEnvironments',
+        'common',
+        'envlayouts'
+    );
+    suite('Conda', () => {
+        test('Conda layout with conda-meta and python binary in the same directory', async () => {
+            const interpreterPath: string = path.join(testLayoutsRoot, 'conda1', 'python.exe');
+            const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+            assert.deepEqual(envType, EnvironmentType.Conda);
+        });
+        test('Conda layout with conda-meta and python binary in a sub directory', async () => {
+            const interpreterPath: string = path.join(testLayoutsRoot, 'conda2', 'bin', 'python');
+            const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+            assert.deepEqual(envType, EnvironmentType.Conda);
+        });
+    });
+
+    suite('Windows Store', () => {
+        let getEnvVar: sinon.SinonStub;
+        const fakeLocalAppDataPath = 'X:\\users\\user\\AppData\\Local';
+        const fakeProgramFilesPath = 'X:\\Program Files';
+        const executable = ['python.exe', 'python3.exe', 'python3.8.exe'];
+        suiteSetup(() => {
+            getEnvVar = sinon.stub(platformApis, 'getEnvironmentVariable');
+            getEnvVar.withArgs('LOCALAPPDATA').returns(fakeLocalAppDataPath);
+            getEnvVar.withArgs('ProgramFiles').returns(fakeProgramFilesPath);
+        });
+        suiteTeardown(() => {
+            getEnvVar.restore();
+        });
+        executable.forEach((exe) => {
+            test(`Path to local app data windows store interpreter (${exe})`, async () => {
+                const interpreterPath = path.join(fakeLocalAppDataPath, 'Microsoft', 'WindowsApps', exe);
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Path to local app data windows store interpreter app sub-directory (${exe})`, async () => {
+                const interpreterPath = path.join(
+                    fakeLocalAppDataPath,
+                    'Microsoft',
+                    'WindowsApps',
+                    'PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0',
+                    exe
+                );
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Path to program files windows store interpreter app sub-directory (${exe})`, async () => {
+                const interpreterPath = path.join(
+                    fakeProgramFilesPath,
+                    'WindowsApps',
+                    'PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0',
+                    exe
+                );
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Local app data not set (${exe})`, async () => {
+                getEnvVar.withArgs('LOCALAPPDATA').returns(undefined);
+                const interpreterPath = path.join(fakeLocalAppDataPath, 'Microsoft', 'WindowsApps', exe);
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Program files app data not set (${exe})`, async () => {
+                getEnvVar.withArgs('ProgramFiles').returns(undefined);
+                const interpreterPath = path.join(
+                    fakeProgramFilesPath,
+                    'WindowsApps',
+                    'PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0',
+                    exe
+                );
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Path using forward slashes (${exe})`, async () => {
+                const interpreterPath = path
+                    .join(fakeLocalAppDataPath, 'Microsoft', 'WindowsApps', exe)
+                    .replace('\\', '/');
+                const envType: EnvironmentType = await identifyEnvironment(interpreterPath);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+            test(`Path using long path style slashes (${exe})`, async () => {
+                const interpreterPath = path
+                    .join(fakeLocalAppDataPath, 'Microsoft', 'WindowsApps', exe)
+                    .replace('\\', '/');
+                const envType: EnvironmentType = await identifyEnvironment(`\\\\?\\${interpreterPath}`);
+                assert.deepEqual(envType, EnvironmentType.WindowsStore);
+            });
+        });
+    });
+});

src/test/pythonEnvironments/common/envlayouts/conda1/conda-meta/history

@@ -0,0 +1 @@
+Usually contains command that was used to create or update the conda environment with time stamps.

src/test/pythonEnvironments/common/envlayouts/conda1/python.exe

@@ -0,0 +1 @@
+Not real python exe

src/test/pythonEnvironments/common/envlayouts/conda2/bin/python

@@ -0,0 +1 @@
+Not a real python binary

src/test/pythonEnvironments/common/envlayouts/conda2/conda-meta/history

@@ -0,0 +1 @@
+Usually contains command that was used to create or update the conda environment with time stamps.