Tests for prompting to install missing ipykernel (#14266)

Author: DonJayamanne

.vscode/launch.json

@@ -177,6 +177,35 @@
             "preLaunchTask": "Compile",
             "skipFiles": ["<node_internals>/**"]
         },
+        {
+            "name": "DataScience tests in VS Code (*.vscode.ts)",
+            "type": "extensionHost",
+            "request": "launch",
+            "runtimeExecutable": "${execPath}",
+            "args": [
+                "${workspaceFolder}/src/test/datascience",
+                "--disable-extensions",
+                "--extensionDevelopmentPath=${workspaceFolder}",
+                "--extensionTestsPath=${workspaceFolder}/out/test"
+            ],
+            "env": {
+                "VSC_PYTHON_CI_TEST_GREP": "", // Modify this to run a subset of the single workspace tests
+                "VSC_PYTHON_CI_TEST_INVERT_GREP": "", // Initialize this to invert the grep (exclude tests with value defined in grep).
+                "CI_PYTHON_PATH": "<PythonPath>", // Initialize this to invert the grep (exclude tests with value defined in grep).
+                "VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE": "true",
+                "TEST_FILES_SUFFIX": "vscode.test"
+            },
+            "stopOnEntry": false,
+            "sourceMaps": true,
+            "outFiles": [
+                "${workspaceFolder}/out/**/*.js",
+                "!${workspaceFolder}/**/node_modules**/*"
+            ],
+            "preLaunchTask": "Compile",
+            "skipFiles": [
+                "<node_internals>/**"
+            ]
+        },
         {
             "name": "Tests (Multiroot, VS Code, *.test.ts)",
             "type": "extensionHost",

build/ci/templates/test_phases.yml

@@ -373,6 +373,23 @@ steps:
       buildPlatform: '$(Agent.Os)-Py$(pythonVersion)'
       buildConfiguration: 'SystemTests'
 
+
+  # Slow DataScience tests with VS Code
+  # Create a venv & register it as a kernel.
+  # These tests are slow hence will only run on linux.
+  # This env will be used to install ipykernel & test for prompts if ipykernel is missing & similar tests.
+  # Ensure this is registered as a kernel.
+  - bash: |
+      python -m venv .venvnokernel
+      source .venvnokernel/bin/activate
+
+      python -m pip install ipykernel
+      python -m ipykernel install --user --name .venvnokernel --display-name .venvnokernel
+      python -m pip uninstall ipykernel --yes
+    displayName: 'Prepare Virtual Env for Kernel Tests'
+    workingDirectory: $(Build.SourcesDirectory)/src/test/datascience
+    condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScienceInVSCode'))
+
   # Set the CI_PYTHON_PATH variable that forces VS Code system tests to use
   # the specified Python interpreter.
   #
@@ -438,18 +455,29 @@ steps:
     env:
       DISPLAY: :10
 
-  # Run the single workspace tests in VS Code Insiders.
+  # Run the single workspace tests for DS in VS Code Insiders.
   - script: |
       npm run testDataScience
     continueOnError: true
     displayName: 'Run DataScience Tests in VSCode Insiders'
-    condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScience'))
+    condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScience'), not(contains(variables['TestsToRun'], 'testDataScienceInVSCode')))
     env:
       DISPLAY: :10
       VSC_PYTHON_CI_TEST_VSC_CHANNEL: 'insiders'
       VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE: 'true'
       TEST_FILES_SUFFIX: 'ds.test'
 
+  # Run the single workspace tests for DS in VS Code Stable.
+  - script: |
+      npm run testDataScienceInVSCode
+    displayName: 'Run DataScience Tests in VSCode Stable'
+    condition: and(succeeded(), contains(variables['TestsToRun'], 'testDataScienceInVSCode'))
+    env:
+      DISPLAY: :10
+      VSC_PYTHON_CI_TEST_VSC_CHANNEL: 'stable'
+      VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE: 'true'
+      TEST_FILES_SUFFIX: 'vscode.test'
+
   # Upload the test results to Azure DevOps to facilitate test reporting in their UX.
   - task: PublishTestResults@2
     displayName: 'Publish single workspace tests results'

build/ci/vscode-python-ci.yaml

@@ -66,6 +66,10 @@ stages:
             'Debugger':
               TestsToRun: 'testDebugger'
               NeedsPythonTestReqs: true
+            'DataScience in VSCode':
+              TestsToRun: 'testDataScienceInVSCode'
+              NeedsPythonTestReqs: true
+              NeedsPythonFunctionalReqs: true
             'Smoke':
               TestsToRun: 'testSmoke'
               NeedsPythonTestReqs: true

build/ci/vscode-python-pr-validation.yaml

@@ -58,6 +58,10 @@ stages:
               TestsToRun: 'testDataScience'
               NeedsPythonTestReqs: true
               NeedsPythonFunctionalReqs: true
+            'DataScience in VSCode':
+              TestsToRun: 'testDataScienceInVSCode'
+              NeedsPythonTestReqs: true
+              NeedsPythonFunctionalReqs: true
             'Smoke':
               TestsToRun: 'testSmoke'
               NeedsPythonTestReqs: true

package.json

@@ -3472,6 +3472,8 @@
         "testJediLSP": "node ./out/test/languageServers/jedi/lspSetup.js && cross-env CODE_TESTS_WORKSPACE=src/test VSC_PYTHON_CI_TEST_GREP='Language Server:' node ./out/test/testBootstrap.js ./out/test/standardTest.js && node ./out/test/languageServers/jedi/lspTeardown.js",
         "pretestDataScience": "node ./out/test/datascience/dsTestSetup.js",
         "testDataScience": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_PYTHON_CI_TEST_VSC_CHANNEL=insiders TEST_FILES_SUFFIX=ds.test VSC_PYTHON_FORCE_LOGGING=1 VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.js ./out/test/standardTest.js",
+        "pretestDataScienceInVSCode": "node ./out/test/datascience/dsTestSetup.js",
+        "testDataScienceInVSCode": "cross-env CODE_TESTS_WORKSPACE=src/test/datascience VSC_PYTHON_CI_TEST_VSC_CHANNEL=stable TEST_FILES_SUFFIX=vscode.test VSC_PYTHON_FORCE_LOGGING=1 VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE=true node ./out/test/testBootstrap.js ./out/test/standardTest.js",
         "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js",
         "testPerformance": "node ./out/test/testBootstrap.js ./out/test/performanceTest.js",
         "testSmoke": "node ./out/test/smokeTest.js",

src/client/datascience/interactive-ipynb/nativeEditorOldWebView.ts

@@ -51,7 +51,7 @@ import {
 import { NativeEditor } from './nativeEditor';
 import { NativeEditorSynchronizer } from './nativeEditorSynchronizer';
 
-enum AskForSaveResult {
+export enum AskForSaveResult {
     Yes,
     No,
     Cancel

src/test/datascience/.vscode/settings.json

@@ -13,7 +13,6 @@
     "python.formatting.provider": "yapf",
     "python.pythonPath": "python",
     "python.experiments.optInto": [
-        "LocalZMQKernel - experiment",
         "NativeNotebook - experiment"
     ],
     // Do not set this to "Microsoft", else it will result in LS being downloaded on CI
@@ -22,8 +21,11 @@
     "python.languageServer": "Jedi",
     // Ensure auto save is off.
     "files.autoSave": "off",
+    //If enabled, ensure we save immediately.
+    "files.autoSaveDelay": 1,
     // We don't want jupyter to start when testing (DS functionality or anything else).
     "python.dataScience.disableJupyterAutoStart": true,
     "python.logging.level": "debug",
-    "python.dataScience.alwaysTrustNotebooks": true // In UI tests we don't want prompts for `Do you want to trust thie nb...`
+    "python.dataScience.alwaysTrustNotebooks": true, // In UI tests we don't want prompts for `Do you want to trust thie nb...`
+    "python.dataScience.useNotebookEditor": true
 }

src/test/datascience/dsTestSetup.ts

@@ -2,64 +2,77 @@
 // Licensed under the MIT License.
 
 import * as fs from 'fs-extra';
+import { applyEdits, ModificationOptions, modify } from 'jsonc-parser';
 import * as path from 'path';
+import { IS_CI_SERVER } from '../ciConstants';
 import { EXTENSION_ROOT_DIR_FOR_TESTS } from '../constants';
-/**
- * Modify package.json to ensure VSC Notebooks have been setup so tests can run.
- * This is required because we modify package.json during runtime, hence we need to do the same thing for tests.
- */
 
-const packageJsonFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'package.json');
-const content = JSON.parse(fs.readFileSync(packageJsonFile).toString());
+const settingsFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src/test/datascience/.vscode/settings.json');
 
-// This code is temporary.
-if (
-    !content.enableProposedApi ||
-    !Array.isArray(content.contributes.notebookOutputRenderer) ||
-    !Array.isArray(content.contributes.notebookProvider)
-) {
-    content.enableProposedApi = true;
-    content.contributes.notebookOutputRenderer = [
-        {
-            viewType: 'jupyter-notebook-renderer',
-            displayName: 'Jupyter Notebook Renderer',
-            mimeTypes: [
-                'application/geo+json',
-                'application/vdom.v1+json',
-                'application/vnd.dataresource+json',
-                'application/vnd.plotly.v1+json',
-                'application/vnd.vega.v2+json',
-                'application/vnd.vega.v3+json',
-                'application/vnd.vega.v4+json',
-                'application/vnd.vega.v5+json',
-                'application/vnd.vegalite.v1+json',
-                'application/vnd.vegalite.v2+json',
-                'application/vnd.vegalite.v3+json',
-                'application/vnd.vegalite.v4+json',
-                'application/x-nteract-model-debug+json',
-                'image/gif',
-                'image/png',
-                'image/jpeg',
-                'text/latex',
-                'text/vnd.plotly.v1+html'
-            ]
-        }
-    ];
-    content.contributes.notebookProvider = [
-        {
-            viewType: 'jupyter-notebook',
-            displayName: 'Jupyter Notebook',
-            selector: [
-                {
-                    filenamePattern: '*.ipynb'
-                }
-            ]
-        }
-    ];
+function updateTestsForNativeNotebooks() {
+    /**
+     * Modify package.json to ensure VSC Notebooks have been setup so tests can run.
+     * This is required because we modify package.json during runtime, hence we need to do the same thing for tests.
+     */
+    const packageJsonFile = path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'package.json');
+    const content = JSON.parse(fs.readFileSync(packageJsonFile).toString());
+
+    // This code is temporary.
+    if (
+        !content.enableProposedApi ||
+        !Array.isArray(content.contributes.notebookOutputRenderer) ||
+        !Array.isArray(content.contributes.notebookProvider)
+    ) {
+        content.enableProposedApi = true;
+        content.contributes.notebookProvider = [
+            {
+                viewType: 'jupyter-notebook',
+                displayName: 'Jupyter Notebook',
+                selector: [
+                    {
+                        filenamePattern: '*.ipynb'
+                    }
+                ]
+            }
+        ];
+    }
+
+    // Update package.json to pick experiments from our custom settings.json file.
+    content.contributes.configuration.properties['python.experiments.optInto'].scope = 'resource';
+    content.contributes.configuration.properties['python.logging.level'].scope = 'resource';
+
+    fs.writeFileSync(packageJsonFile, JSON.stringify(content, undefined, 4));
+
+    updateSettings(true);
 }
 
-// Update package.json to pick experiments from our custom settings.json file.
-content.contributes.configuration.properties['python.experiments.optInto'].scope = 'resource';
-content.contributes.configuration.properties['python.logging.level'].scope = 'resource';
+function updateSettings(useNativeNotebooks: boolean) {
+    const modificationOptions: ModificationOptions = {
+        formattingOptions: {
+            tabSize: 4,
+            insertSpaces: true
+        }
+    };
+    let settingsJson = fs.readFileSync(settingsFile).toString();
+    const experiments = useNativeNotebooks ? ['NativeNotebook - experiment'] : [];
+    const autoSave = useNativeNotebooks ? 'off' : 'afterDelay';
+
+    settingsJson = applyEdits(
+        settingsJson,
+        modify(settingsJson, ['python.experiments.optInto'], experiments, modificationOptions)
+    );
+    settingsJson = applyEdits(settingsJson, modify(settingsJson, ['files.autoSave'], autoSave, modificationOptions));
+
+    fs.writeFileSync(settingsFile, settingsJson);
+}
+function updateTestsForOldNotebooks() {
+    updateSettings(false);
+}
 
-fs.writeFileSync(packageJsonFile, JSON.stringify(content, undefined, 4));
+if (!IS_CI_SERVER) {
+    // Noop.
+} else if (process.env.VSC_PYTHON_CI_TEST_VSC_CHANNEL === 'insiders') {
+    updateTestsForNativeNotebooks();
+} else {
+    updateTestsForOldNotebooks();
+}

src/test/datascience/helpers.ts

@@ -4,7 +4,19 @@
 'use strict';
 
 import { noop } from 'lodash';
+import * as path from 'path';
+import { Uri } from 'vscode';
+import { ICommandManager } from '../../client/common/application/types';
 import { IDataScienceSettings } from '../../client/common/types';
+import { Commands } from '../../client/datascience/constants';
+import {
+    AskForSaveResult,
+    NativeEditorOldWebView
+} from '../../client/datascience/interactive-ipynb/nativeEditorOldWebView';
+import { INotebookEditorProvider } from '../../client/datascience/types';
+import { IServiceContainer } from '../../client/ioc/types';
+import { CommandSource } from '../../client/testing/common/constants';
+import { waitForCondition } from '../common';
 
 // The default base set of data science settings to use
 export function defaultDataScienceSettings(): IDataScienceSettings {
@@ -61,3 +73,23 @@ export function writeDiffSnapshot(_snapshot: any, _prefix: string) {
     // snapshotCounter += 1;
     // fs.writeFile(file, JSON.stringify(diff), { encoding: 'utf-8' }).ignoreErrors();
 }
+
+export async function openNotebook(serviceContainer: IServiceContainer, ipynbFile: string, ignoreSaving = true) {
+    const cmd = serviceContainer.get<ICommandManager>(ICommandManager);
+    await cmd.executeCommand(Commands.OpenNotebook, Uri.file(ipynbFile), CommandSource.commandPalette);
+    const editorProvider = serviceContainer.get<INotebookEditorProvider>(INotebookEditorProvider);
+    await waitForCondition(
+        async () =>
+            editorProvider.editors.length > 0 &&
+            !!editorProvider.activeEditor &&
+            editorProvider.activeEditor.file.fsPath.endsWith(path.basename(ipynbFile)),
+        30_000,
+        'Notebook not opened'
+    );
+
+    if (ignoreSaving && editorProvider.activeEditor && editorProvider.activeEditor instanceof NativeEditorOldWebView) {
+        // We don't care about changes, no need to save them.
+        // tslint:disable-next-line: no-any
+        (editorProvider.activeEditor as any).askForSave = () => Promise.resolve(AskForSaveResult.No);
+    }
+}