Know posix path locator (#14184)

* Known posix path locator

* Add tests

* Add iterEnvs test

* More tests

* Address comments

Author: karthiknadig

src/client/pythonEnvironments/common/externalDependencies.ts

@@ -61,7 +61,16 @@ export function getGlobalPersistentStore<T>(key: string): IPersistentStore<T> {
 export async function getFileInfo(filePath: string): Promise<{ctime:number, mtime:number}> {
     const data = await fsapi.lstat(filePath);
     return {
-        ctime: data.ctime.getUTCDate(),
-        mtime: data.mtime.getUTCDate(),
+        ctime: data.ctime.valueOf(),
+        mtime: data.mtime.valueOf(),
     };
 }
+
+export async function resolveSymbolicLink(filepath:string): Promise<string> {
+    const stats = await fsapi.lstat(filepath);
+    if (stats.isSymbolicLink()) {
+        const link = await fsapi.readlink(filepath);
+        return resolveSymbolicLink(link);
+    }
+    return filepath;
+}

src/client/pythonEnvironments/common/posixUtils.ts

@@ -0,0 +1,61 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import * as path from 'path';
+import { getPathEnvironmentVariable } from '../../common/utils/platform';
+
+/**
+ * Checks if a given path ends with python*.exe
+ * @param {string} interpreterPath : Path to python interpreter.
+ * @returns {boolean} : Returns true if the path matches pattern for windows python executable.
+ */
+export function isPosixPythonBin(interpreterPath:string): boolean {
+    /**
+     * This Reg-ex matches following file names:
+     * python
+     * python3
+     * python38
+     * python3.8
+     */
+    const posixPythonBinPattern = /^python(\d+(\.\d+)?)?$/;
+
+    return posixPythonBinPattern.test(path.basename(interpreterPath));
+}
+
+export async function commonPosixBinPaths(): Promise<string[]> {
+    const searchPaths = (getPathEnvironmentVariable() || '')
+        .split(path.delimiter)
+        .filter((p) => p.length > 0);
+
+    const paths: string[] = Array.from(new Set(
+        [
+            '/bin',
+            '/etc',
+            '/lib',
+            '/lib/x86_64-linux-gnu',
+            '/lib64',
+            '/sbin',
+            '/snap/bin',
+            '/usr/bin',
+            '/usr/games',
+            '/usr/include',
+            '/usr/lib',
+            '/usr/lib/x86_64-linux-gnu',
+            '/usr/lib64',
+            '/usr/libexec',
+            '/usr/local',
+            '/usr/local/bin',
+            '/usr/local/etc',
+            '/usr/local/games',
+            '/usr/local/lib',
+            '/usr/local/sbin',
+            '/usr/sbin',
+            '/usr/share',
+            '~/.local/bin',
+        ].concat(searchPaths),
+    ));
+
+    const exists = await Promise.all(paths.map((p) => fsapi.pathExists(p)));
+    return paths.filter((_, index) => exists[index]);
+}

src/client/pythonEnvironments/discovery/locators/services/posixKnownPathsLocator.ts

@@ -0,0 +1,91 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import * as path from 'path';
+import { traceError, traceInfo } from '../../../../common/logger';
+
+import { Architecture } from '../../../../common/utils/platform';
+import {
+    PythonEnvInfo, PythonEnvKind, PythonReleaseLevel, PythonVersion,
+} from '../../../base/info';
+import { parseVersion } from '../../../base/info/pythonVersion';
+import { ILocator, IPythonEnvsIterator } from '../../../base/locator';
+import { PythonEnvsWatcher } from '../../../base/watcher';
+import { getFileInfo, resolveSymbolicLink } from '../../../common/externalDependencies';
+import { commonPosixBinPaths, isPosixPythonBin } from '../../../common/posixUtils';
+
+async function getPythonBinFromKnownPaths(): Promise<string[]> {
+    const knownPaths = await commonPosixBinPaths();
+    const pythonBins:Set<string> = new Set();
+    // eslint-disable-next-line no-restricted-syntax
+    for (const knownPath of knownPaths) {
+        // eslint-disable-next-line no-await-in-loop
+        const files = (await fsapi.readdir(knownPath))
+            .map((filename:string) => path.join(knownPath, filename))
+            .filter(isPosixPythonBin);
+
+        // eslint-disable-next-line no-restricted-syntax
+        for (const file of files) {
+            // Ensure that we have a collection of unique global binaries by
+            // resolving all symlinks to the target binaries.
+            try {
+                // eslint-disable-next-line no-await-in-loop
+                const resolvedBin = await resolveSymbolicLink(file);
+                pythonBins.add(resolvedBin);
+                traceInfo(`Found: ${file} --> ${resolvedBin}`);
+            } catch (ex) {
+                traceError('Failed to resolve symbolic link: ', ex);
+            }
+        }
+    }
+
+    return Array.from(pythonBins);
+}
+
+export class PosixKnownPathsLocator extends PythonEnvsWatcher implements ILocator {
+    private kind: PythonEnvKind = PythonEnvKind.OtherGlobal;
+
+    public iterEnvs(): IPythonEnvsIterator {
+        const buildEnvInfo = (bin:string) => this.buildEnvInfo(bin);
+        const iterator = async function* () {
+            const exes = await getPythonBinFromKnownPaths();
+            yield* exes.map(buildEnvInfo);
+        };
+        return iterator();
+    }
+
+    public resolveEnv(env: string | PythonEnvInfo): Promise<PythonEnvInfo | undefined> {
+        const executablePath = typeof env === 'string' ? env : env.executable.filename;
+        return this.buildEnvInfo(executablePath);
+    }
+
+    private async buildEnvInfo(bin:string): Promise<PythonEnvInfo> {
+        let version:PythonVersion;
+        try {
+            version = parseVersion(path.basename(bin));
+        } catch (ex) {
+            traceError(`Failed to parse version from path: ${bin}`, ex);
+            version = {
+                major: -1,
+                minor: -1,
+                micro: -1,
+                release: { level: PythonReleaseLevel.Final, serial: -1 },
+                sysVersion: undefined,
+            };
+        }
+        return {
+            name: '',
+            location: '',
+            kind: this.kind,
+            executable: {
+                filename: bin,
+                sysPrefix: '',
+                ...(await getFileInfo(bin)),
+            },
+            version,
+            arch: Architecture.Unknown,
+            distro: { org: '' },
+        };
+    }
+}

src/client/pythonEnvironments/discovery/locators/services/windowsStoreLocator.ts

@@ -3,7 +3,7 @@
 
 import * as fsapi from 'fs-extra';
 import * as path from 'path';
-import { traceWarning } from '../../../../common/logger';
+import { traceError, traceWarning } from '../../../../common/logger';
 import { Architecture, getEnvironmentVariable } from '../../../../common/utils/platform';
 import {
     PythonEnvInfo, PythonEnvKind, PythonReleaseLevel, PythonVersion,
@@ -138,7 +138,8 @@ export class WindowsStoreLocator extends PythonEnvsWatcher implements ILocator {
         let version:PythonVersion;
         try {
             version = parseVersion(path.basename(exe));
-        } catch (e) {
+        } catch (ex) {
+            traceError(`Failed to parse version from path: ${exe}`, ex);
             version = {
                 major: 3,
                 minor: -1,

src/test/pythonEnvironments/common/envlayouts/posixroot/location1/python

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location1/python3

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location2/python

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location2/python37

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location2/python38

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location3/python3.7

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/common/envlayouts/posixroot/location3/python3.8

@@ -0,0 +1 @@
+Not a real binary

src/test/pythonEnvironments/discovery/locators/envTestUtils.ts

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as assert from 'assert';
+import { zip } from 'lodash';
+import { PythonEnvInfo } from '../../../../client/pythonEnvironments/base/info';
+
+export function assertEnvEqual(actual:PythonEnvInfo | undefined, expected: PythonEnvInfo | undefined):void {
+    assert.notStrictEqual(actual, undefined);
+    assert.notStrictEqual(expected, undefined);
+
+    if (actual) {
+    // ensure ctime and mtime are greater than -1
+        assert.ok(actual?.executable.ctime > -1);
+        assert.ok(actual?.executable.mtime > -1);
+
+        // No need to match these, so reset them
+        actual.executable.ctime = -1;
+        actual.executable.mtime = -1;
+
+        assert.deepStrictEqual(actual, expected);
+    }
+}
+
+export function assertEnvsEqual(
+    actualEnvs:(PythonEnvInfo | undefined)[],
+    expectedEnvs: (PythonEnvInfo | undefined)[],
+):void{
+    zip(actualEnvs, expectedEnvs).forEach((value) => {
+        const [actual, expected] = value;
+        assertEnvEqual(actual, expected);
+    });
+}

src/test/pythonEnvironments/discovery/locators/posixKnownPathsLocator.unit.test.ts

@@ -0,0 +1,148 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as path from 'path';
+import * as sinon from 'sinon';
+import * as platformApis from '../../../../client/common/utils/platform';
+import {
+    PythonEnvInfo, PythonEnvKind, PythonReleaseLevel, PythonVersion,
+} from '../../../../client/pythonEnvironments/base/info';
+import { InterpreterInformation } from '../../../../client/pythonEnvironments/base/info/interpreter';
+import { parseVersion } from '../../../../client/pythonEnvironments/base/info/pythonVersion';
+import { getEnvs } from '../../../../client/pythonEnvironments/base/locatorUtils';
+import { PosixKnownPathsLocator } from '../../../../client/pythonEnvironments/discovery/locators/services/posixKnownPathsLocator';
+import { TEST_LAYOUT_ROOT } from '../../common/commonTestConstants';
+import { assertEnvEqual, assertEnvsEqual } from './envTestUtils';
+
+suite('Posix Known Path Locator', () => {
+    let getPathEnvVar: sinon.SinonStub;
+    const testPosixKnownPathsRoot = path.join(TEST_LAYOUT_ROOT, 'posixroot');
+
+    const testLocation1 = path.join(testPosixKnownPathsRoot, 'location1');
+    const testLocation2 = path.join(testPosixKnownPathsRoot, 'location2');
+    const testLocation3 = path.join(testPosixKnownPathsRoot, 'location3');
+
+    const testFileData:Map<string, string[]> = new Map();
+
+    testFileData.set(testLocation1, ['python', 'python3']);
+    testFileData.set(testLocation2, ['python', 'python37', 'python38']);
+    testFileData.set(testLocation3, ['python3.7', 'python3.8']);
+
+    function createExpectedInterpreterInfo(
+        executable: string,
+        sysVersion?: string,
+        sysPrefix?: string,
+        versionStr?:string,
+    ): InterpreterInformation {
+        let version:PythonVersion;
+        try {
+            version = parseVersion(versionStr ?? path.basename(executable));
+            if (sysVersion) {
+                version.sysVersion = sysVersion;
+            }
+        } catch (e) {
+            version = {
+                major: -1,
+                minor: -1,
+                micro: -1,
+                release: { level: PythonReleaseLevel.Final, serial: -1 },
+                sysVersion,
+            };
+        }
+        return {
+            version,
+            arch: platformApis.Architecture.Unknown,
+            executable: {
+                filename: executable,
+                sysPrefix: sysPrefix ?? '',
+                ctime: -1,
+                mtime: -1,
+            },
+        };
+    }
+
+    setup(() => {
+        getPathEnvVar = sinon.stub(platformApis, 'getPathEnvironmentVariable');
+    });
+    teardown(() => {
+        getPathEnvVar.restore();
+    });
+    test('iterEnvs(): get python bin from known test roots', async () => {
+        const testLocations = [testLocation1, testLocation2, testLocation3];
+        getPathEnvVar.returns(testLocations.join(path.delimiter));
+
+        const envs:PythonEnvInfo[] = [];
+        testLocations.forEach((location) => {
+            const binaries = testFileData.get(location);
+            if (binaries) {
+                binaries.forEach((binary) => {
+                    envs.push({
+                        name: '',
+                        location: '',
+                        kind: PythonEnvKind.OtherGlobal,
+                        distro: { org: '' },
+                        ...createExpectedInterpreterInfo(path.join(location, binary)),
+                    });
+                });
+            }
+        });
+        const expectedEnvs = envs.sort((a, b) => a.executable.filename.localeCompare(b.executable.filename));
+
+        const locator = new PosixKnownPathsLocator();
+        const actualEnvs = (await getEnvs(locator.iterEnvs()))
+            .filter((e) => e.executable.filename.indexOf('posixroot') > 0)
+            .sort((a, b) => a.executable.filename.localeCompare(b.executable.filename));
+        assertEnvsEqual(actualEnvs, expectedEnvs);
+    });
+    test('resolveEnv(string)', async () => {
+        const pythonPath = path.join(testLocation1, 'python');
+        const expected = {
+            name: '',
+            location: '',
+            kind: PythonEnvKind.OtherGlobal,
+            distro: { org: '' },
+            ...createExpectedInterpreterInfo(pythonPath),
+        };
+
+        const locator = new PosixKnownPathsLocator();
+        const actual = await locator.resolveEnv(pythonPath);
+        assertEnvEqual(actual, expected);
+    });
+    test('resolveEnv(PythonEnvInfo)', async () => {
+        const pythonPath = path.join(testLocation1, 'python');
+        const expected = {
+
+            name: '',
+            location: '',
+            kind: PythonEnvKind.OtherGlobal,
+            distro: { org: '' },
+            ...createExpectedInterpreterInfo(pythonPath),
+        };
+
+        // Partially filled in env info object
+        const input:PythonEnvInfo = {
+            name: '',
+            location: '',
+            kind: PythonEnvKind.Unknown,
+            distro: { org: '' },
+            arch: platformApis.Architecture.Unknown,
+            executable: {
+                filename: pythonPath,
+                sysPrefix: '',
+                ctime: -1,
+                mtime: -1,
+            },
+            version: {
+                major: -1,
+                minor: -1,
+                micro: -1,
+                release: { level: PythonReleaseLevel.Final, serial: -1 },
+            },
+        };
+
+        const locator = new PosixKnownPathsLocator();
+        const actual = await locator.resolveEnv(input);
+
+        assertEnvEqual(actual, expected);
+    });
+});