Refactored exporting notebooks to python script (#12232)

* some work done

* linking done

* quickpick menu working

* added command pallete commands

* moved to commandRegister

* started work on file saving

* file saving started

* some work done

* linking done

* quickpick menu working

* added command pallete commands

* moved to commandRegister

* started work on file saving

* file saving started

* handled some saving edge cases

* moved to new file

* more work

* started on exportbase

* saving almost done

* removed save prompt

* fixed

* working in new files

* changes to export base

* small fixes

* export to python workin

* added tests

* removed unused injections

* started work on dependency checking

* cleaned up

* removed broken lines

* attempt to fix failing tests

* attempt to fix failing tests 2

* attempt to fix failing tests 3

* unmodified launch.json

* created export commands

* refactoring and working on dependency checks

* split functionallity into multiple classes

* hopefully fixed tests

* almost done file opening for python

* cleaned up

* made test

* removed notebookeditorprovider as dependency

* fixed small bugs

* fixed small bugs

* removed sleep

* fixed default file name

* fixed filename when saving

* added instructions for adding new export method

* added missing lines to package.nls.json

* python export behaviour changed to original

* fixed test

* added readme instructions

* fixed test

* fixed other test

* fixed lint

* fixed one test

* fixed final test and made export command hidden

* moved type defs into types.ts

* localized text

* added types file

* added localized test to package.nls.json

Author: techwithtim

.vscode/launch.json

@@ -204,7 +204,7 @@
             "env": {
                 "VSC_PYTHON_CI_TEST_GREP": "", // Modify this to run a subset of the single workspace tests
                 "VSC_PYTHON_CI_TEST_INVERT_GREP": "", // Initialize this to invert the grep (exclude tests with value defined in grep).
-                
+
                 "VSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE": "true",
                 "TEST_FILES_SUFFIX": "ds.test"
             },

package.json

@@ -59,6 +59,9 @@
         "onCommand:python.viewTestOutput",
         "onCommand:python.viewOutput",
         "onCommand:python.datascience.viewJupyterOutput",
+        "onCommand:python.datascience.exportAsPythonScript",
+        "onCommand:python.datascience.exportToHTML",
+        "onCommand:python.datascience.exportToPDF",
         "onCommand:python.selectAndRunTestMethod",
         "onCommand:python.selectAndDebugTestMethod",
         "onCommand:python.selectAndRunTestFile",
@@ -352,6 +355,11 @@
                 "title": "%python.command.python.datascience.viewJupyterOutput.title%",
                 "category": "Python"
             },
+            {
+                "command": "python.datascience.exportAsPythonScript",
+                "title": "%python.command.python.datascience.exportAsPythonScript.title%",
+                "category": "Python"
+            },
             {
                 "command": "python.datascience.selectJupyterInterpreter",
                 "title": "%python.command.python.datascience.selectJupyterInterpreter.title%",
@@ -844,6 +852,12 @@
                 }
             ],
             "commandPalette": [
+                {
+                    "command": "python.datascience.exportAsPythonScript",
+                    "title": "%python.command.python.datascience.exportAsPythonScript.title%",
+                    "category": "Python",
+                    "when": "python.datascience.isnativeactive && python.datascience.featureenabled"
+                },
                 {
                     "command": "python.switchOffInsidersChannel",
                     "title": "%python.command.python.switchOffInsidersChannel.title%",

package.nls.json

@@ -17,6 +17,9 @@
     "python.command.python.viewOutput.title": "Show Output",
     "python.command.python.viewTestOutput.title": "Show Test Output",
     "python.command.python.datascience.viewJupyterOutput.title": "Show Jupyter Output",
+    "python.command.python.datascience.exportAsPythonScript.title": "Export as Python Script",
+    "python.command.python.datascience.exportToHTML.title": "Export to HTML",
+    "python.command.python.datascience.exportToPDF.title": "Export to PDF",
     "python.command.python.viewLanguageServerOutput.title": "Show Language Server Output",
     "python.command.python.selectAndRunTestMethod.title": "Run Test Method ...",
     "python.command.python.selectAndDebugTestMethod.title": "Debug Test Method ...",
@@ -107,6 +110,9 @@
     "DataScience.exportDialogFailed": "Failed to export notebook. {0}",
     "DataScience.exportOpenQuestion": "Open in browser",
     "DataScience.exportOpenQuestion1": "Open in editor",
+    "DataScience.notebookExportAs" : "Convert and save to a python script",
+    "DataScience.exportAsQuickPickPlaceholder" : "Export As...",
+    "DataScience.exportPythonQuickPickLabel" : "Python Script",
     "DataScience.collapseInputTooltip": "Collapse input block",
     "DataScience.collapseVariableExplorerTooltip": "Hide variables active in jupyter kernel",
     "DataScience.collapseVariableExplorerLabel": "Variables",

src/client/common/application/commands.ts

@@ -6,7 +6,7 @@
 import { CancellationToken, Position, TextDocument, Uri } from 'vscode';
 import { Commands as LSCommands } from '../../activation/languageServer/constants';
 import { Commands as DSCommands } from '../../datascience/constants';
-import { INotebook } from '../../datascience/types';
+import { INotebook, INotebookModel } from '../../datascience/types';
 import { CommandSource } from '../../testing/common/constants';
 import { TestFunction, TestsToRun } from '../../testing/common/types';
 import { TestDataItem, TestWorkspaceFolder } from '../../testing/types';
@@ -168,6 +168,10 @@ export interface ICommandNameArgumentTypeMapping extends ICommandNameWithoutArgu
     [DSCommands.RunCurrentCellAndAddBelow]: [string];
     [DSCommands.ScrollToCell]: [string, string];
     [DSCommands.ViewJupyterOutput]: [];
+    [DSCommands.ExportAsPythonScript]: [INotebookModel];
+    [DSCommands.ExportToHTML]: [INotebookModel];
+    [DSCommands.ExportToPDF]: [INotebookModel];
+    [DSCommands.Export]: [INotebookModel];
     [DSCommands.SwitchJupyterKernel]: [INotebook | undefined, 'raw' | 'jupyter'];
     [DSCommands.SelectJupyterCommandLine]: [undefined | Uri];
     [DSCommands.SaveNotebookNonCustomEditor]: [Uri];

src/client/common/utils/localize.ts

@@ -560,6 +560,7 @@ export namespace DataScience {
     export const clearAllOutput = localize('DataScience.clearAllOutput', 'Clear All Output');
     export const interruptKernelStatus = localize('DataScience.interruptKernelStatus', 'Interrupting IPython Kernel');
     export const exportCancel = localize('DataScience.exportCancel', 'Cancel');
+    export const exportPythonQuickPickLabel = localize('DataScience.exportPythonQuickPickLabel', 'Python Script');
     export const restartKernelAfterInterruptMessage = localize(
         'DataScience.restartKernelAfterInterruptMessage',
         'Interrupting the kernel timed out. Do you want to restart the kernel instead? All variables will be lost.'
@@ -764,11 +765,9 @@ export namespace DataScience {
         'DataScience.remoteDebuggerNotSupported',
         'Debugging while attached to a remote server is not currently supported.'
     );
-    export const exportAsPythonFileTooltip = localize(
-        'DataScience.exportAsPythonFileTooltip',
-        'Convert and save to a python script'
-    );
+    export const notebookExportAs = localize('DataScience.notebookExportAs', 'Convert and save to a python script');
     export const exportAsPythonFileTitle = localize('DataScience.exportAsPythonFileTitle', 'Save As Python File');
+    export const exportAsQuickPickPlaceholder = localize('DataScience.exportAsQuickPickPlaceholder', 'Export As...');
     export const runCell = localize('DataScience.runCell', 'Run cell');
     export const deleteCell = localize('DataScience.deleteCell', 'Delete cell');
     export const moveCellUp = localize('DataScience.moveCellUp', 'Move cell up');

src/client/datascience/commands/commandRegistry.ts

@@ -21,6 +21,7 @@ import {
     INotebookEditorProvider
 } from '../types';
 import { JupyterCommandLineSelectorCommand } from './commandLineSelector';
+import { ExportCommands } from './exportCommands';
 import { KernelSwitcherCommand } from './kernelSwitcher';
 import { JupyterServerSelectorCommand } from './serverSelector';
 
@@ -44,7 +45,8 @@ export class CommandRegistry implements IDisposable {
         @inject(IApplicationShell) private appShell: IApplicationShell,
         @inject(IOutputChannel) @named(JUPYTER_OUTPUT_CHANNEL) private jupyterOutput: IOutputChannel,
         @inject(IStartPage) private startPage: IStartPage,
-        @inject(IExperimentService) private readonly expService: IExperimentService
+        @inject(IExperimentService) private readonly expService: IExperimentService,
+        @inject(ExportCommands) private readonly exportCommand: ExportCommands
     ) {
         this.disposables.push(this.serverSelectedCommand);
         this.disposables.push(this.kernelSwitcherCommand);
@@ -53,6 +55,7 @@ export class CommandRegistry implements IDisposable {
         this.commandLineCommand.register();
         this.serverSelectedCommand.register();
         this.kernelSwitcherCommand.register();
+        this.exportCommand.register();
         this.registerCommand(Commands.RunAllCells, this.runAllCells);
         this.registerCommand(Commands.RunCell, this.runCell);
         this.registerCommand(Commands.RunCurrentCell, this.runCurrentCell);

src/client/datascience/commands/exportCommands.ts

@@ -0,0 +1,98 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { inject, injectable } from 'inversify';
+import { QuickPickItem, QuickPickOptions } from 'vscode';
+import { getLocString } from '../../../datascience-ui/react-common/locReactSide';
+import { ICommandNameArgumentTypeMapping } from '../../common/application/commands';
+import { IApplicationShell, ICommandManager } from '../../common/application/types';
+import { IDisposable } from '../../common/types';
+import { DataScience } from '../../common/utils/localize';
+import { Commands } from '../constants';
+import { ExportManager } from '../export/exportManager';
+import { ExportFormat, IExportManager } from '../export/types';
+import { INotebookEditorProvider, INotebookModel } from '../types';
+
+interface IExportQuickPickItem extends QuickPickItem {
+    handler(): void;
+}
+
+@injectable()
+export class ExportCommands implements IDisposable {
+    private readonly disposables: IDisposable[] = [];
+    constructor(
+        @inject(ICommandManager) private readonly commandManager: ICommandManager,
+        @inject(IExportManager) private exportManager: ExportManager,
+        @inject(IApplicationShell) private readonly applicationShell: IApplicationShell,
+        @inject(INotebookEditorProvider) private readonly notebookProvider: INotebookEditorProvider
+    ) {}
+    public register() {
+        this.registerCommand(Commands.ExportAsPythonScript, (model) => this.export(model, ExportFormat.python));
+        this.registerCommand(Commands.ExportToHTML, (model) => this.export(model, ExportFormat.html));
+        this.registerCommand(Commands.ExportToPDF, (model) => this.export(model, ExportFormat.pdf));
+        this.registerCommand(Commands.Export, (model) => this.export(model));
+    }
+
+    public dispose() {
+        this.disposables.forEach((d) => d.dispose());
+    }
+
+    private registerCommand<
+        E extends keyof ICommandNameArgumentTypeMapping,
+        U extends ICommandNameArgumentTypeMapping[E]
+        // tslint:disable-next-line: no-any
+    >(command: E, callback: (...args: U) => any) {
+        const disposable = this.commandManager.registerCommand(command, callback, this);
+        this.disposables.push(disposable);
+    }
+
+    private async export(model: INotebookModel, exportMethod?: ExportFormat) {
+        if (!model) {
+            // if no model was passed then this was called from the command pallete,
+            // so we need to get the active editor
+            const activeEditor = this.notebookProvider.activeEditor;
+            if (!activeEditor || !activeEditor.model) {
+                return;
+            }
+            model = activeEditor.model;
+        }
+
+        if (exportMethod) {
+            await this.exportManager.export(exportMethod, model);
+        } else {
+            // if we don't have an export method we need to ask for one and display the
+            // quickpick menu
+            const pickedItem = await this.showExportQuickPickMenu(model).then((item) => item);
+            if (pickedItem !== undefined) {
+                pickedItem.handler();
+            }
+        }
+    }
+
+    private getExportQuickPickItems(model: INotebookModel): IExportQuickPickItem[] {
+        return [
+            {
+                label: DataScience.exportPythonQuickPickLabel(),
+                picked: true,
+                handler: () => this.commandManager.executeCommand(Commands.ExportAsPythonScript, model)
+            }
+            //{ label: 'HTML', picked: false, handler: () => this.commandManager.executeCommand(Commands.ExportToHTML) },
+            //{ label: 'PDF', picked: false, handler: () => this.commandManager.executeCommand(Commands.ExportToPDF) }
+        ];
+    }
+
+    private async showExportQuickPickMenu(model: INotebookModel): Promise<IExportQuickPickItem | undefined> {
+        const items = this.getExportQuickPickItems(model);
+
+        const options: QuickPickOptions = {
+            ignoreFocusOut: false,
+            matchOnDescription: true,
+            matchOnDetail: true,
+            placeHolder: getLocString('DataScience.exportAsQuickPickPlaceholder', 'Export As...')
+        };
+
+        return this.applicationShell.showQuickPick(items, options);
+    }
+}

src/client/datascience/constants.ts

@@ -79,6 +79,10 @@ export namespace Commands {
     export const ScrollToCell = 'python.datascience.scrolltocell';
     export const CreateNewNotebook = 'python.datascience.createnewnotebook';
     export const ViewJupyterOutput = 'python.datascience.viewJupyterOutput';
+    export const ExportAsPythonScript = 'python.datascience.exportAsPythonScript';
+    export const ExportToHTML = 'python.datascience.exportToHTML';
+    export const ExportToPDF = 'python.datascience.exportToPDF';
+    export const Export = 'python.datascience.export';
     export const SaveNotebookNonCustomEditor = 'python.datascience.notebookeditor.save';
     export const SaveAsNotebookNonCustomEditor = 'python.datascience.notebookeditor.saveAs';
     export const OpenNotebookNonCustomEditor = 'python.datascience.notebookeditor.open';

src/client/datascience/export/README.md

@@ -0,0 +1,13 @@
+
+## TO ADD A NEW EXPORT METHOD
+1. Create a new command in src/client/datascience/constants
+2. Register the command in this file
+3. Add an item to the quick pick menu for your new export method from inside the getExportQuickPickItems() method (in this file).
+4. Add a new command to the command pallete in package.json (optional)
+5. Declare and add your file extensions inside exportManagerFilePicker
+6. Declare and add your export method inside exportManager
+7. Create an injectable class that implements IExport and register it in src/client/datascience/serviceRegistry
+8. Implement the export method on your new class
+9. Inject the class inside exportManager
+10. Add a case for your new export method and call the export method of your new class with the appropriate arguments
+11. Add telementry and status messages

src/client/datascience/export/exportManager.ts

@@ -0,0 +1,72 @@
+import { inject, injectable, named } from 'inversify';
+import { Uri } from 'vscode';
+import { IFileSystem, TemporaryFile } from '../../common/platform/types';
+import { IDataScienceErrorHandler, INotebookModel } from '../types';
+import { IExportManagerFilePicker } from './exportManagerFilePicker';
+import { ExportFormat, IExport, IExportManager } from './types';
+
+@injectable()
+export class ExportManager implements IExportManager {
+    constructor(
+        @inject(IExport) @named(ExportFormat.pdf) private readonly exportToPDF: IExport,
+        @inject(IExport) @named(ExportFormat.html) private readonly exportToHTML: IExport,
+        @inject(IExport) @named(ExportFormat.python) private readonly exportToPython: IExport,
+        @inject(IFileSystem) private readonly fileSystem: IFileSystem,
+        @inject(IDataScienceErrorHandler) private readonly errorHandler: IDataScienceErrorHandler,
+        @inject(IExportManagerFilePicker) private readonly filePicker: IExportManagerFilePicker
+    ) {}
+
+    public async export(format: ExportFormat, model: INotebookModel): Promise<Uri | undefined> {
+        // need to add telementry
+        let target;
+        if (format !== ExportFormat.python) {
+            target = await this.filePicker.getExportFileLocation(format, model.file);
+            if (!target) {
+                return;
+            }
+        } else {
+            target = Uri.file((await this.fileSystem.createTemporaryFile('.py')).filePath);
+        }
+
+        const tempFile = await this.makeTemporaryFile(model);
+        if (!tempFile) {
+            return; // error making temp file
+        }
+
+        const source = Uri.file(tempFile.filePath);
+        try {
+            switch (format) {
+                case ExportFormat.python:
+                    await this.exportToPython.export(source, target);
+                    break;
+
+                case ExportFormat.pdf:
+                    await this.exportToPDF.export(source, target);
+                    break;
+
+                case ExportFormat.html:
+                    await this.exportToHTML.export(source, target);
+                    break;
+                default:
+                    break;
+            }
+        } finally {
+            tempFile.dispose();
+        }
+
+        return target;
+    }
+
+    private async makeTemporaryFile(model: INotebookModel): Promise<TemporaryFile | undefined> {
+        let tempFile: TemporaryFile | undefined;
+        try {
+            tempFile = await this.fileSystem.createTemporaryFile('.ipynb');
+            const content = model ? model.getContent() : '';
+            await this.fileSystem.writeFile(tempFile.filePath, content, 'utf-8');
+        } catch (e) {
+            await this.errorHandler.handleError(e);
+        }
+
+        return tempFile;
+    }
+}

src/client/datascience/export/exportManagerDependencyChecker.ts

@@ -0,0 +1,29 @@
+import { inject, injectable } from 'inversify';
+import { Uri } from 'vscode';
+import * as localize from '../../common/utils/localize';
+import { IJupyterExecution, IJupyterInterpreterDependencyManager, INotebookModel } from '../types';
+import { ExportManager } from './exportManager';
+import { ExportFormat, IExportManager } from './types';
+
+@injectable()
+export class ExportManagerDependencyChecker implements IExportManager {
+    constructor(
+        @inject(ExportManager) private readonly manager: IExportManager,
+        @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution,
+        @inject(IJupyterInterpreterDependencyManager)
+        private readonly dependencyManager: IJupyterInterpreterDependencyManager
+    ) {}
+
+    public async export(format: ExportFormat, model: INotebookModel): Promise<Uri | undefined> {
+        // Before we try the import, see if we don't support it, if we don't give a chance to install dependencies
+        if (!(await this.jupyterExecution.isImportSupported())) {
+            await this.dependencyManager.installMissingDependencies();
+        }
+
+        if (await this.jupyterExecution.isImportSupported()) {
+            return this.manager.export(format, model);
+        } else {
+            throw new Error(localize.DataScience.jupyterNbConvertNotSupported());
+        }
+    }
+}