Load widget scripts from CDN and/or local python interpreter (#10987)

* Address sonar issues
* Fix linter
* Fixes
* Better way to pas array buffer
* Added comments
* Oops

Author: DonJayamanne

package.json

@@ -1636,10 +1636,22 @@
                     "description": "Allows a user to import a jupyter notebook into a python file anytime one is opened.",
                     "scope": "resource"
                 },
-                "python.dataScience.loadWidgetScriptsFromThirdPartySource": {
-                    "type": "boolean",
-                    "default": false,
-                    "description": "Enables loading of scripts files for Widgets (ipywidgest, bqplot, beakerx, ipyleaflet, etc) from https://unpkg.com.",
+                "python.dataScience.widgetScriptSources": {
+                    "type": "array",
+                    "default": [],
+                    "items": {
+                        "type": "string",
+                        "enum": [
+                            "jsdelivr.com",
+                            "unpkg.com"
+                        ],
+                        "enumDescriptions": [
+                            "Loads widget (javascript) scripts from https://www.jsdelivr.com/",
+                            "Loads widget (javascript) scripts from https://unpkg.com/"
+                        ]
+                    },
+                    "uniqueItems": true,
+                    "markdownDescription": "Defines the location and order of the sources where scripts files for Widgets are downloaded from (e.g. ipywidgest, bqplot, beakerx, ipyleaflet, etc). Not selecting any of these could result in widgets not rendering or function correctly. See [here](https://aka.ms/PVSCIPyWidgets) for more information. Once updated you will need to restart the Kernel.",
                     "scope": "machine"
                 },
                 "python.dataScience.gatherRules": {

package.nls.json

@@ -145,6 +145,7 @@
     "Common.reload": "Reload",
     "Common.moreInfo": "More Info",
     "Common.and": "and",
+    "Common.ok": "Ok",
     "Common.install": "Install",
     "Common.learnMore": "Learn more",
     "OutputChannelNames.languageServer": "Python Language Server",
@@ -464,6 +465,7 @@
     "DataScience.jupyterSelectURIRemoteDetail": "Specify the URI of an existing server",
     "DataScience.gatherQuality": "Did gather work as desired?",
     "DataScience.loadClassFailedWithNoInternet": "Error loading {0}:{1}. Internet connection required for loading 3rd party widgets.",
-    "DataScience.loadThirdPartyWidgetScriptsDisabled": "Loading of Widgets is disabled by default. Click <a href='https://command:python.datascience.loadWidgetScriptsFromThirdPartySource'>here</a> to enable the setting 'python.dataScience.loadWidgetScriptsFromThirdPartySource'. Once enabled you will need to restart the Kernel",
-    "DataScience.loadThirdPartyWidgetScriptsPostEnabled": "Please restart the Kernel when changing the setting 'loadWidgetScriptsFromThirdPartySource'."
+    "DataScience.useCDNForWidgets": "Widgets require us to download supporting files from a 3rd party website. Click [here](https://aka.ms/PVSCIPyWidgets) for more information.",
+    "DataScience.loadThirdPartyWidgetScriptsPostEnabled": "Please restart the Kernel when changing the setting 'python.dataScience.widgetScriptSources'.",
+    "DataScience.enableCDNForWidgetsSetting": "Widgets require us to download supporting files from a 3rd party website. Click <a href='https://command:python.datascience.enableLoadingWidgetScriptsFromThirdPartySource'>here</a> to enable this or click <a href='https://aka.ms/PVSCIPyWidgets'>here</a> for more information. (Error loading {0}:{1})."
 }

src/client/common/application/types.ts

@@ -1045,6 +1045,18 @@ export type WebPanelMessage = {
 // Wraps the VS Code webview panel
 export const IWebPanel = Symbol('IWebPanel');
 export interface IWebPanel {
+    /**
+     * Convert a uri for the local file system to one that can be used inside webviews.
+     *
+     * Webviews cannot directly load resources from the workspace or local file system using `file:` uris. The
+     * `asWebviewUri` function takes a local `file:` uri and converts it into a uri that can be used inside of
+     * a webview to load the same resource:
+     *
+     * ```ts
+     * webview.html = `<img src="${webview.asWebviewUri(vscode.Uri.file('/Users/codey/workspace/cat.gif'))}">`
+     * ```
+     */
+    asWebviewUri(localResource: Uri): Uri;
     setTitle(val: string): void;
     /**
      * Makes the webpanel show up.

src/client/common/application/webPanels/webPanel.ts

@@ -71,6 +71,12 @@ export class WebPanel implements IWebPanel {
             this.panel.dispose();
         }
     }
+    public asWebviewUri(localResource: Uri) {
+        if (!this.panel) {
+            throw new Error('WebView not initialized, too early to get a Uri');
+        }
+        return this.panel?.webview.asWebviewUri(localResource);
+    }
 
     public isVisible(): boolean {
         return this.panel ? this.panel.visible : false;
@@ -161,7 +167,7 @@ export class WebPanel implements IWebPanel {
                 <meta name="theme" content="${Identifiers.GeneratedThemeName}"/>
                 <title>VS Code Python React UI</title>
                 <base href="${uriBase}${uriBase.endsWith('/') ? '' : '/'}"/>
-                <link rel="stylesheet" href="${rootPath}/../common/node_modules/font-awesome/css/font-awesome.min.css">                
+                <link rel="stylesheet" href="${rootPath}/../common/node_modules/font-awesome/css/font-awesome.min.css">
                 </head>
             <body>
                 <noscript>You need to enable JavaScript to run this app.</noscript>

src/client/common/net/httpClient.ts

@@ -5,7 +5,7 @@
 
 import { inject, injectable } from 'inversify';
 import { parse, ParseError } from 'jsonc-parser';
-import * as requestTypes from 'request';
+import type * as requestTypes from 'request';
 import { IHttpClient } from '../../common/types';
 import { IServiceContainer } from '../../ioc/types';
 import { IWorkspaceService } from '../application/types';
@@ -26,7 +26,7 @@ export class HttpClient implements IHttpClient {
     }
 
     public async getJSON<T>(uri: string, strict: boolean = true): Promise<T> {
-        const body = await this.getBody(uri);
+        const body = await this.getContents(uri);
         return this.parseBodyToJSON(body, strict);
     }
 
@@ -44,7 +44,21 @@ export class HttpClient implements IHttpClient {
         }
     }
 
-    public async getBody(uri: string): Promise<string> {
+    public async exists(uri: string): Promise<boolean> {
+        // tslint:disable-next-line:no-require-imports
+        const request = require('request') as typeof requestTypes;
+        return new Promise<boolean>((resolve) => {
+            try {
+                request
+                    .get(uri, this.requestOptions)
+                    .on('response', (response) => resolve(response.statusCode === 200))
+                    .on('error', () => resolve(false));
+            } catch {
+                resolve(false);
+            }
+        });
+    }
+    private async getContents(uri: string): Promise<string> {
         // tslint:disable-next-line:no-require-imports
         const request = require('request') as typeof requestTypes;
         return new Promise<string>((resolve, reject) => {

src/client/common/types.ts

@@ -393,9 +393,11 @@ export interface IDataScienceSettings {
     variableQueries: IVariableQuery[];
     disableJupyterAutoStart?: boolean;
     jupyterCommandLineArguments: string[];
-    loadWidgetScriptsFromThirdPartySource?: boolean;
+    widgetScriptSources: WidgetCDNs[];
 }
 
+export type WidgetCDNs = 'unpkg.com' | 'jsdelivr.com';
+
 export const IConfigurationService = Symbol('IConfigurationService');
 export interface IConfigurationService {
     getSettings(resource?: Uri): IPythonSettings;
@@ -466,6 +468,10 @@ export interface IHttpClient {
      * @param strict Set `false` to allow trailing comma and comments in the JSON, defaults to `true`
      */
     getJSON<T>(uri: string, strict?: boolean): Promise<T>;
+    /**
+     * Returns the url is valid (i.e. return status code of 200).
+     */
+    exists(uri: string): Promise<boolean>;
 }
 
 export const IExtensionContext = Symbol('ExtensionContext');

src/client/common/utils/localize.ts

@@ -63,6 +63,7 @@ export namespace Common {
     export const bannerLabelNo = localize('Common.bannerLabelNo', 'No');
     export const canceled = localize('Common.canceled', 'Canceled');
     export const cancel = localize('Common.cancel', 'Cancel');
+    export const ok = localize('Common.ok', 'Ok');
     export const gotIt = localize('Common.gotIt', 'Got it!');
     export const install = localize('Common.install', 'Install');
     export const loadingExtension = localize('Common.loadingPythonExtension', 'Python extension loading...');
@@ -838,7 +839,15 @@ export namespace DataScience {
     );
     export const loadThirdPartyWidgetScriptsPostEnabled = localize(
         'DataScience.loadThirdPartyWidgetScriptsPostEnabled',
-        "Once you have updated the setting 'loadWidgetScriptsFromThirdPartySource' you will need to restart the Kernel."
+        "Please restart the Kernel when changing the setting 'python.dataScience.widgetScriptSources'."
+    );
+    export const useCDNForWidgets = localize(
+        'DataScience.useCDNForWidgets',
+        'Widgets require us to download supporting files from a 3rd party website. Click [here](https://aka.ms/PVSCIPyWidgets) for more information.'
+    );
+    export const enableCDNForWidgetsSetting = localize(
+        'DataScience.enableCDNForWidgetsSetting',
+        "Widgets require us to download supporting files from a 3rd party website. Click <a href='https://command:python.datascience.enableLoadingWidgetScriptsFromThirdPartySource'>here</a> to enable this or click <a href='https://aka.ms/PVSCIPyWidgets'>here</a> for more information. (Error loading {0}:{1})."
     );
 }
 

src/client/common/utils/serializers.ts

@@ -31,8 +31,12 @@ export function serializeDataViews(buffers: undefined | (ArrayBuffer | ArrayBuff
                 // tslint:disable-next-line: no-any
             } as any);
         } else {
+            // Do not use `Array.apply`, it will not work for large arrays.
+            // Nodejs will throw `stackoverflow` exceptions.
+            // Else following ipynb fails https://github.com/K3D-tools/K3D-jupyter/blob/821a59ed88579afaafababd6291e8692d70eb088/examples/camera_manipulation.ipynb
+            // Yet another case where 99% can work, but 1% can fail when testing.
             // tslint:disable-next-line: no-any
-            newBufferView.push(Array.apply(null, new Uint8Array(item as any) as any) as any);
+            newBufferView.push([...new Uint8Array(item as any)]);
         }
     }
 

src/client/datascience/commands/commandRegistry.ts

@@ -106,14 +106,14 @@ export class CommandRegistry implements IDisposable {
     }
 
     private enableLoadingWidgetScriptsFromThirdParty(): void {
-        if (this.configService.getSettings(undefined).datascience.loadWidgetScriptsFromThirdPartySource) {
+        if (this.configService.getSettings(undefined).datascience.widgetScriptSources.length > 0) {
             return;
         }
         // Update the setting and once updated, notify user to restart kernel.
         this.configService
             .updateSetting(
-                'dataScience.loadWidgetScriptsFromThirdPartySource',
-                true,
+                'dataScience.widgetScriptSources',
+                ['jsdelivr.com', 'unpkg.com'],
                 undefined,
                 ConfigurationTarget.Global
             )

src/client/datascience/constants.ts

@@ -82,7 +82,8 @@ export namespace Commands {
     export const SaveAsNotebookNonCustomEditor = 'python.datascience.notebookeditor.saveAs';
     export const OpenNotebookNonCustomEditor = 'python.datascience.notebookeditor.open';
     export const GatherQuality = 'python.datascience.gatherquality';
-    export const EnableLoadingWidgetsFrom3rdPartySource = 'python.datascience.loadWidgetScriptsFromThirdPartySource';
+    export const EnableLoadingWidgetsFrom3rdPartySource =
+        'python.datascience.enableLoadingWidgetScriptsFromThirdPartySource';
 }
 
 export namespace CodeLensCommands {
@@ -292,6 +293,13 @@ export enum Telemetry {
     IPyWidgetLoadSuccess = 'DS_INTERNAL.IPYWIDGET_LOAD_SUCCESS',
     IPyWidgetLoadFailure = 'DS_INTERNAL.IPYWIDGET_LOAD_FAILURE',
     IPyWidgetLoadDisabled = 'DS_INTERNAL.IPYWIDGET_LOAD_DISABLED',
+    HashedIPyWidgetNameUsed = 'DS_INTERNAL.IPYWIDGET_USED_BY_USER',
+    HashedIPyWidgetNameDiscovered = 'DS_INTERNAL.IPYWIDGET_DISCOVERED',
+    HashedIPyWidgetScriptDiscoveryError = 'DS_INTERNAL.IPYWIDGET_DISCOVERY_ERRORED',
+    DiscoverIPyWidgetNamesLocalPerf = 'DS_INTERNAL.IPYWIDGET_TEST_AVAILABILITY_ON_LOCAL',
+    DiscoverIPyWidgetNamesCDNPerf = 'DS_INTERNAL.IPYWIDGET_TEST_AVAILABILITY_ON_CDN',
+    IPyWidgetPromptToUseCDN = 'DS_INTERNAL.IPYWIDGET_PROMPT_TO_USE_CDN',
+    IPyWidgetPromptToUseCDNSelection = 'DS_INTERNAL.IPYWIDGET_PROMPT_TO_USE_CDN_SELECTION',
     IPyWidgetOverhead = 'DS_INTERNAL.IPYWIDGET_OVERHEAD',
     IPyWidgetRenderFailure = 'DS_INTERNAL.IPYWIDGET_RENDER_FAILURE'
 }

src/client/datascience/interactive-common/interactiveBase.ts

@@ -177,6 +177,12 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
 
         // For each listener sign up for their post events
         this.listeners.forEach((l) => l.postMessage((e) => this.postMessageInternal(e.message, e.payload)));
+        // Channel for listeners to send messages to the interactive base.
+        this.listeners.forEach((l) => {
+            if (l.postInternalMessage) {
+                l.postInternalMessage((e) => this.onMessage(e.message, e.payload));
+            }
+        });
 
         // Tell each listener our identity. Can't do it here though as were in the constructor for the base class
         setTimeout(() => {
@@ -204,6 +210,12 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
     // tslint:disable-next-line: no-any no-empty cyclomatic-complexity max-func-body-length
     public onMessage(message: string, payload: any) {
         switch (message) {
+            case InteractiveWindowMessages.ConvertUriForUseInWebViewRequest:
+                const request = payload as Uri;
+                const response = { request, response: this.asWebviewUri(request) };
+                this.postMessageToListeners(InteractiveWindowMessages.ConvertUriForUseInWebViewResponse, response);
+                break;
+
             case InteractiveWindowMessages.Started:
                 // Send the first settings message
                 this.onDataScienceSettingsChanged().ignoreErrors();

src/client/datascience/interactive-common/interactiveWindowTypes.ts

@@ -10,10 +10,10 @@ import {
     CommonActionType,
     IAddCellAction,
     ILoadIPyWidgetClassFailureAction,
-    LoadIPyWidgetClassDisabledAction,
     LoadIPyWidgetClassLoadAction
 } from '../../../datascience-ui/interactive-common/redux/reducers/types';
 import { PythonInterpreter } from '../../interpreter/contracts';
+import { WidgetScriptSource } from '../ipywidgets/types';
 import { LiveKernelModel } from '../jupyter/kernels/types';
 import { CssMessages, IGetCssRequest, IGetCssResponse, IGetMonacoThemeRequest, SharedMessages } from '../messages';
 import { IGetMonacoThemeResponse } from '../monacoMessages';
@@ -53,6 +53,8 @@ export enum InteractiveWindowMessages {
     DoSave = 'DoSave',
     SendInfo = 'send_info',
     Started = 'started',
+    ConvertUriForUseInWebViewRequest = 'ConvertUriForUseInWebViewRequest',
+    ConvertUriForUseInWebViewResponse = 'ConvertUriForUseInWebViewResponse',
     AddedSysInfo = 'added_sys_info',
     RemoteAddCode = 'remote_add_code',
     RemoteReexecuteCode = 'remote_reexecute_code',
@@ -111,13 +113,22 @@ export enum InteractiveWindowMessages {
     UpdateDisplayData = 'update_display_data',
     IPyWidgetLoadSuccess = 'ipywidget_load_success',
     IPyWidgetLoadFailure = 'ipywidget_load_failure',
-    IPyWidgetLoadDisabled = 'ipywidget_load_disabled',
     IPyWidgetRenderFailure = 'ipywidget_render_failure'
 }
 
 export enum IPyWidgetMessages {
     IPyWidgets_Ready = 'IPyWidgets_Ready',
     IPyWidgets_onRestartKernel = 'IPyWidgets_onRestartKernel',
+    IPyWidgets_onKernelChanged = 'IPyWidgets_onKernelChanged',
+    IPyWidgets_updateRequireConfig = 'IPyWidgets_updateRequireConfig',
+    /**
+     * UI sends a request to extension to determine whether we have the source for any of the widgets.
+     */
+    IPyWidgets_WidgetScriptSourceRequest = 'IPyWidgets_WidgetScriptSourceRequest',
+    /**
+     * Extension sends response to the request with yes/no.
+     */
+    IPyWidgets_WidgetScriptSourceResponse = 'IPyWidgets_WidgetScriptSourceResponse',
     IPyWidgets_msg = 'IPyWidgets_msg',
     IPyWidgets_binary_msg = 'IPyWidgets_binary_msg',
     IPyWidgets_msg_handled = 'IPyWidgets_msg_handled',
@@ -477,8 +488,11 @@ export type NotebookModelChange =
 // Map all messages to specific payloads
 export class IInteractiveWindowMapping {
     public [IPyWidgetMessages.IPyWidgets_kernelOptions]: KernelSocketOptions;
+    public [IPyWidgetMessages.IPyWidgets_WidgetScriptSourceRequest]: { moduleName: string; moduleVersion: string };
+    public [IPyWidgetMessages.IPyWidgets_WidgetScriptSourceResponse]: WidgetScriptSource;
     public [IPyWidgetMessages.IPyWidgets_Ready]: never | undefined;
     public [IPyWidgetMessages.IPyWidgets_onRestartKernel]: never | undefined;
+    public [IPyWidgetMessages.IPyWidgets_onKernelChanged]: never | undefined;
     public [IPyWidgetMessages.IPyWidgets_registerCommTarget]: string;
     // tslint:disable-next-line: no-any
     public [IPyWidgetMessages.IPyWidgets_binary_msg]: { id: string; data: any };
@@ -589,6 +603,7 @@ export class IInteractiveWindowMapping {
     public [InteractiveWindowMessages.UpdateDisplayData]: KernelMessage.IUpdateDisplayDataMsg;
     public [InteractiveWindowMessages.IPyWidgetLoadSuccess]: LoadIPyWidgetClassLoadAction;
     public [InteractiveWindowMessages.IPyWidgetLoadFailure]: ILoadIPyWidgetClassFailureAction;
-    public [InteractiveWindowMessages.IPyWidgetLoadDisabled]: LoadIPyWidgetClassDisabledAction;
+    public [InteractiveWindowMessages.ConvertUriForUseInWebViewRequest]: Uri;
+    public [InteractiveWindowMessages.ConvertUriForUseInWebViewResponse]: { request: Uri; response: Uri };
     public [InteractiveWindowMessages.IPyWidgetRenderFailure]: Error;
 }

src/client/datascience/interactive-common/linkProvider.ts

@@ -19,7 +19,7 @@ const LineQueryRegex = /line=(\d+)/;
 // in a markdown cell using the syntax: https://command:[my.vscode.command].
 const linkCommandWhitelist = [
     'python.datascience.gatherquality',
-    'python.datascience.loadWidgetScriptsFromThirdPartySource'
+    'python.datascience.enableLoadingWidgetScriptsFromThirdPartySource'
 ];
 
 // tslint:disable: no-any