Default cell language for native notebooks

.github/workflows/pr_datascience.yml

@@ -32,7 +32,7 @@ jobs:
         # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
         # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
         os: [ubuntu-latest]
-        python: [3.8] 
+        python: [3.8]
         test-suite: [group1, group2, group3, group4]
     steps:
       - name: Checkout
@@ -149,8 +149,140 @@ jobs:
       - name: Publish Test Report
         uses: scacap/action-surefire-report@v1
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}       
-          report_paths: ${{ env.TEST_RESULTS_GLOB }} 
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: ${{ env.TEST_RESULTS_GLOB }}
           check_name: Functional Test Report
         if: steps.test_functional_group.outcome == 'failure' && failure()
 
+  testsInVSCode:
+    name: Tests in VS Code
+    runs-on: ${{ matrix.os }}
+    if: github.repository == 'microsoft/vscode-python'
+    strategy:
+      fail-fast: false
+      matrix:
+        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
+        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
+        os: [ubuntu-latest]
+        python: [3.8]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Use Python ${{matrix.python}}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{matrix.python}}
+
+      - name: Upgrade pip
+        run: python -m pip install -U pip
+
+      - name: Use Node ${{env.NODE_VERSION}}
+        uses: actions/[email protected]
+        with:
+          node-version: ${{env.NODE_VERSION}}
+
+      # Start caching
+
+      # Cache Python Dependencies.
+      # Caching (https://github.com/actions/cache/blob/main/examples.md#python---pip
+      - name: Cache pip on linux
+        uses: actions/cache@v2
+        if: matrix.os == 'ubuntu-latest'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      - name: Cache pip on mac
+        uses: actions/cache@v2
+        if: matrix.os == 'macos-latest'
+        with:
+          path: ~/Library/Caches/pip
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      - name: Cache pip on windows
+        uses: actions/cache@v2
+        if: matrix.os == 'windows-latest'
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      # Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)
+      - name: Cache npm on linux/mac
+        uses: actions/cache@v2
+        if: matrix.os != 'windows-latest'
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Get npm cache directory
+        if: matrix.os == 'windows-latest'
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$(npm config get cache)"
+      - name: Cache npm on windows
+        uses: actions/cache@v2
+        if: matrix.os == 'windows-latest'
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Cache compiled TS files
+        id: out-cache
+        uses: actions/cache@v2
+        with:
+          path: ./out
+          key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}
+
+      # For faster/better builds of sdists.
+      - run: python -m pip install wheel
+        shell: bash
+
+      # debugpy is not shipped, only installed for local tests.
+      # In production, we get debugpy from python extension.
+      - name: Install functional test requirements
+        run: |
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r ./requirements.txt
+          python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt
+          python ./pythonFiles/install_debugpy.py
+          python -m pip install numpy
+          python -m pip install --upgrade jupyter
+          python -m pip install --upgrade -r build/test-requirements.txt
+          python -m pip install --upgrade -r ./build/ipython-test-requirements.txt
+          python -m pip install --upgrade -r ./build/conda-functional-requirements.txt
+          python -m ipykernel install --user
+        # This step is slow.
+
+      - name: Install dependencies (npm ci)
+        run: npm ci --prefer-offline
+        # This step is slow.
+
+      - name: Compile if not cached
+        run: npx gulp prePublishNonBundle
+
+      - name: Run VSCode tests
+        run: npm run test:testDataScienceInVSCode
+        env:
+          VSCODE_PYTHON_ROLLING: 1
+          VSC_PYTHON_FORCE_LOGGING: 1
+          VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE: 'true'
+        id: test_functional
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: ${{ env.TEST_RESULTS_GLOB }}
+          check_name: VSCode Test Report
+        if: steps.test_functional.outcome == 'failure' && failure()
+

package.json

@@ -3344,6 +3344,16 @@
                 "extensions": [
                     ".ipynb"
                 ]
+            },
+            {
+                "id": "julia",
+                "aliases": [
+                    "Julia",
+                    "julia"
+                ],
+                "extensions": [
+                    ".jl"
+                ]
             }
         ],
         "grammars": [

src/client/common/application/notebook.ts

@@ -52,6 +52,11 @@ export class VSCodeNotebook implements IVSCodeNotebook {
             ? this.notebook.onDidCloseNotebookDocument
             : new EventEmitter<NotebookDocument>().event;
     }
+    public get onDidSaveNotebookDocument(): Event<NotebookDocument> {
+        return this.canUseNotebookApi
+            ? this.notebook.onDidSaveNotebookDocument
+            : new EventEmitter<NotebookDocument>().event;
+    }
     public get notebookDocuments(): ReadonlyArray<NotebookDocument> {
         return this.canUseNotebookApi ? this.notebook.notebookDocuments : [];
     }

src/client/common/application/types.ts

@@ -1540,6 +1540,7 @@ export interface IVSCodeNotebook {
     readonly notebookDocuments: ReadonlyArray<NotebookDocument>;
     readonly onDidOpenNotebookDocument: Event<NotebookDocument>;
     readonly onDidCloseNotebookDocument: Event<NotebookDocument>;
+    readonly onDidSaveNotebookDocument: Event<NotebookDocument>;
     readonly onDidChangeActiveNotebookEditor: Event<NotebookEditor | undefined>;
     readonly onDidChangeNotebookDocument: Event<NotebookCellChangedEvent>;
     readonly notebookEditors: Readonly<NotebookEditor[]>;

src/client/datascience/common.ts

@@ -131,11 +131,16 @@ export function traceCellResults(prefix: string, results: ICell[]) {
 }
 
 export function translateKernelLanguageToMonaco(kernelLanguage: string): string {
-    // The only known translation is C# to csharp at the moment
-    if (kernelLanguage === 'C#' || kernelLanguage === 'c#') {
-        return 'csharp';
+    // At the moment these are the only translations.
+    // python, julia, r, javascript, powershell, etc can be left as is.
+    switch (kernelLanguage.toLowerCase()) {
+        case 'c#':
+            return 'csharp';
+        case 'f#':
+            return 'fsharp';
+        default:
+            return kernelLanguage.toLowerCase();
     }
-    return kernelLanguage.toLowerCase();
 }
 
 export function generateNewNotebookUri(

src/client/datascience/jupyter/kernels/helpers.ts

@@ -10,6 +10,7 @@ import { JupyterKernelSpec } from './jupyterKernelSpec';
 const NamedRegexp = require('named-js-regexp') as typeof import('named-js-regexp');
 
 // tslint:disable-next-line: no-require-imports
+import { nbformat } from '@jupyterlab/coreutils';
 import cloneDeep = require('lodash/cloneDeep');
 import { PYTHON_LANGUAGE } from '../../../common/constants';
 import { ReadWrite } from '../../../common/types';
@@ -133,6 +134,20 @@ export function getKernelConnectionLanguage(kernelConnection?: KernelConnectionM
         : undefined;
     return model?.language || kernelSpec?.language;
 }
+export function getLanguageInNotebookMetadata(metadata?: nbformat.INotebookMetadata): string | undefined {
+    if (!metadata) {
+        return;
+    }
+    // If kernel spec is defined & we have a language in that, then use that information.
+    // tslint:disable-next-line: no-any
+    const kernelSpec: IJupyterKernelSpec | undefined = metadata.kernelspec as any;
+    // When a kernel spec is stored in ipynb, the `language` of the kernel spec is also saved.
+    // Unfortunately there's no strong typing for this.
+    if (kernelSpec?.language) {
+        return kernelSpec.language;
+    }
+    return metadata.language_info?.name;
+}
 // Create a default kernelspec with the given display name
 export function createDefaultKernelSpec(interpreter?: PythonEnvironment): IJupyterKernelSpec {
     // This creates a default kernel spec. When launched, 'python' argument will map to using the interpreter

src/client/datascience/jupyter/kernels/kernelSelector.ts

@@ -32,7 +32,7 @@ import {
     INotebookProviderConnection,
     KernelInterpreterDependencyResponse
 } from '../../types';
-import { createDefaultKernelSpec, getDisplayNameOrNameOfKernelConnection } from './helpers';
+import { createDefaultKernelSpec, getDisplayNameOrNameOfKernelConnection, isPythonKernelConnection } from './helpers';
 import { KernelSelectionProvider } from './kernelSelections';
 import { KernelService } from './kernelService';
 import {
@@ -521,12 +521,7 @@ export class KernelSelector implements IKernelSelectionUsage {
         }
 
         // First use our kernel finder to locate a kernelspec on disk
-        const kernelSpec = await this.kernelFinder.findKernelSpec(
-            resource,
-            notebookMetadata?.kernelspec,
-            cancelToken,
-            ignoreDependencyCheck
-        );
+        const kernelSpec = await this.kernelFinder.findKernelSpec(resource, notebookMetadata, cancelToken);
         const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);
         if (!kernelSpec && !activeInterpreter) {
             return;
@@ -542,11 +537,16 @@ export class KernelSelector implements IKernelSelectionUsage {
             // Locate the interpreter that matches our kernelspec
             const interpreter = await this.kernelService.findMatchingInterpreter(kernelSpec, cancelToken);
 
-            if (interpreter) {
+            const connectionInfo: KernelSpecConnectionMetadata = {
+                kind: 'startUsingKernelSpec',
+                kernelSpec,
+                interpreter
+            };
+            // Install missing depednencies only if we're dealing with a Python kernel.
+            if (interpreter && isPythonKernelConnection(connectionInfo)) {
                 await this.installDependenciesIntoInterpreter(interpreter, ignoreDependencyCheck, cancelToken);
             }
-
-            return { kind: 'startUsingKernelSpec', kernelSpec, interpreter };
+            return connectionInfo;
         }
     }
 

src/client/datascience/kernel-launcher/kernelFinder.ts

@@ -61,16 +61,59 @@ export class KernelFinder implements IKernelFinder {
     @captureTelemetry(Telemetry.KernelFinderPerf)
     public async findKernelSpec(
         resource: Resource,
-        kernelSpecMetadata?: nbformat.IKernelspecMetadata
+        notebookMetadata?: nbformat.INotebookMetadata
     ): Promise<IJupyterKernelSpec | undefined> {
         await this.readCache();
-        let foundKernel: IJupyterKernelSpec | undefined;
 
+        const searchBasedOnKernelSpecMetadata = this.findKernelSpecBasedOnKernelSpecMetadata(
+            resource,
+            notebookMetadata && notebookMetadata.kernelspec ? notebookMetadata.kernelspec : undefined
+        );
+
+        if (!notebookMetadata || notebookMetadata.kernelspec || !notebookMetadata.language_info?.name) {
+            return searchBasedOnKernelSpecMetadata;
+        }
+
+        // If given a language, then find based on language else revert to default behaviour.
+        const searchBasedOnLanguage = await this.findKernelSpecBasedOnLanguage(
+            resource,
+            notebookMetadata.language_info.name
+        );
+        // If none found based on language, then return the default.s
+        return searchBasedOnLanguage || searchBasedOnKernelSpecMetadata;
+    }
+    // Search all our local file system locations for installed kernel specs and return them
+    public async listKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
+        if (!resource) {
+            // We need a resource to search for related kernel specs
+            return [];
+        }
+
+        // Get an id for the workspace folder, if we don't have one, use the fsPath of the resource
+        const workspaceFolderId = this.workspaceService.getWorkspaceFolderIdentifier(resource, resource.fsPath);
+
+        // If we have not already searched for this resource, then generate the search
+        if (!this.workspaceToKernels.has(workspaceFolderId)) {
+            this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource));
+        }
+
+        this.writeCache().ignoreErrors();
+
+        // ! as the has and set above verify that we have a return here
+        return this.workspaceToKernels.get(workspaceFolderId)!;
+    }
+
+    private async findKernelSpecBasedOnKernelSpecMetadata(
+        resource: Resource,
+        kernelSpecMetadata?: nbformat.IKernelspecMetadata
+    ) {
         const kernelName = kernelSpecMetadata?.name;
+        if (!kernelName) {
+            return;
+        }
 
-        if (kernelSpecMetadata && kernelName) {
+        try {
             let kernelSpec = await this.searchCache(kernelName);
-
             if (kernelSpec) {
                 return kernelSpec;
             }
@@ -79,7 +122,6 @@ export class KernelFinder implements IKernelFinder {
             kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource);
 
             if (kernelSpec) {
-                this.writeCache().ignoreErrors();
                 return kernelSpec;
             }
 
@@ -94,33 +136,15 @@ export class KernelFinder implements IKernelFinder {
                 result = both[0] ? both[0] : both[1];
             }
 
-            foundKernel = result;
+            return result;
+        } finally {
+            this.writeCache().ignoreErrors();
         }
-
-        this.writeCache().ignoreErrors();
-
-        return foundKernel;
     }
 
-    // Search all our local file system locations for installed kernel specs and return them
-    public async listKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
-        if (!resource) {
-            // We need a resource to search for related kernel specs
-            return [];
-        }
-
-        // Get an id for the workspace folder, if we don't have one, use the fsPath of the resource
-        const workspaceFolderId = this.workspaceService.getWorkspaceFolderIdentifier(resource, resource.fsPath);
-
-        // If we have not already searched for this resource, then generate the search
-        if (!this.workspaceToKernels.has(workspaceFolderId)) {
-            this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource));
-        }
-
-        this.writeCache().ignoreErrors();
-
-        // ! as the has and set above verify that we have a return here
-        return this.workspaceToKernels.get(workspaceFolderId)!;
+    private async findKernelSpecBasedOnLanguage(resource: Resource, language: string) {
+        const specs = await this.listKernelSpecs(resource);
+        return specs.find((item) => item.language.toLowerCase() === language.toLowerCase());
     }
 
     private async findResourceKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {