Ds/greazer/fix gather webpack (#11109)

* Support custom gather spec locations

* Adjust gather->script generation

* Interactive Window Gather

* Created GatherToScript command and made the
interactive window always use that when gathering.

* Don't show gather if cell is error

* No gathering markdown

* Add a new folder to Gather spec path

Author: greazer

build/webpack/webpack.extension.config.js

@@ -67,9 +67,18 @@ const config = {
     externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir],
     plugins: [
         ...common.getDefaultPlugins('extension'),
-        // Copy pdfkit after extension builds. webpack can't handle pdfkit.
-        new removeFilesWebpackPlugin({
-            after: { include: ['./out/client/node_modules/pdfkit/js/pdfkit.standalone.*'] }
+        // Copy gather spec files into a known location for the webpacked extension
+        new FileManagerPlugin({
+            onEnd: [
+                {
+                    copy: [
+                        {
+                            source: './node_modules/@msrvida/python-program-analysis/dist/es5/specs/*.yaml',
+                            destination: './out/client/gatherSpecs'
+                        }
+                    ]
+                }
+            ]
         }),
         new copyWebpackPlugin([
             {

package-lock.json

@@ -1655,68 +1655,6 @@
                     "markdownDescription": "Defines the location and order of the sources where scripts files for Widgets are downloaded from (e.g. ipywidgest, bqplot, beakerx, ipyleaflet, etc). Not selecting any of these could result in widgets not rendering or function correctly. See [here](https://aka.ms/PVSCIPyWidgets) for more information. Once updated you will need to restart the Kernel.",
                     "scope": "machine"
                 },
-                "python.dataScience.gatherRules": {
-                    "type": "array",
-                    "default": [
-                        {
-                            "objectName": "df",
-                            "functionName": "head",
-                            "doesNotModify": [
-                                "OBJECT"
-                            ]
-                        },
-                        {
-                            "objectName": "df",
-                            "functionName": "describe",
-                            "doesNotModify": [
-                                "OBJECT"
-                            ]
-                        },
-                        {
-                            "objectName": "df",
-                            "functionName": "tail",
-                            "doesNotModify": [
-                                "OBJECT"
-                            ]
-                        },
-                        {
-                            "functionName": "print",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        },
-                        {
-                            "functionName": "KMeans",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        },
-                        {
-                            "functionName": "scatter",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        },
-                        {
-                            "functionName": "fit",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        },
-                        {
-                            "functionName": "sum",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        },
-                        {
-                            "functionName": "len",
-                            "doesNotModify": [
-                                "ARGUMENTS"
-                            ]
-                        }
-                    ]
-                },
                 "python.dataScience.askForLargeDataFrames": {
                     "type": "boolean",
                     "default": true,
@@ -1941,6 +1879,12 @@
                     "description": "Python Insiders Only: If experimental gather feature is enabled, gather code to a python script rather than a notebook.",
                     "scope": "resource"
                 },
+                "python.dataScience.gatherSpecPath": {
+                    "type": "string",
+                    "default": "",
+                    "description": "Python Insiders Only: If experimental gather feature is enabled, this setting specifies a folder that contains additional or replacement spec files used for analysis.",
+                    "scope": "resource"
+                },
                 "python.dataScience.codeLenses": {
                     "type": "string",
                     "default": "python.datascience.runcell,  python.datascience.runallcellsabove, python.datascience.debugcell",

package.json

@@ -327,12 +327,6 @@ export interface IAnalysisSettings {
     readonly logLevel: LogLevel;
 }
 
-interface IGatherRule {
-    objectName?: string;
-    functionName: string;
-    doesNotModify: string[] | number[];
-}
-
 export interface IVariableQuery {
     language: string;
     query: string;
@@ -356,7 +350,7 @@ export interface IDataScienceSettings {
     maxOutputSize: number;
     enableGather?: boolean;
     gatherToScript?: boolean;
-    gatherRules?: IGatherRule[];
+    gatherSpecPath?: string;
     sendSelectionToInteractiveWindow: boolean;
     markdownRegularExpression: string;
     codeRegularExpression: string;

src/client/common/types.ts

@@ -1,8 +1,10 @@
 import * as ppatypes from '@msrvida-python-program-analysis';
 import { inject, injectable } from 'inversify';
+import * as path from 'path';
 import * as uuid from 'uuid/v4';
 import { IApplicationShell, ICommandManager } from '../../common/application/types';
 import { traceInfo } from '../../common/logger';
+import { IFileSystem } from '../../common/platform/types';
 import { IConfigurationService, IDisposableRegistry } from '../../common/types';
 import * as localize from '../../common/utils/localize';
 // tslint:disable-next-line: no-duplicate-imports
@@ -19,12 +21,14 @@ export class GatherProvider implements IGatherProvider {
     private _executionSlicer: ppatypes.ExecutionLogSlicer<ppatypes.Cell> | undefined;
     private dataflowAnalyzer: ppatypes.DataflowAnalyzer | undefined;
     private _enabled: boolean;
+    private initPromise: Promise<void>;
 
     constructor(
         @inject(IConfigurationService) private configService: IConfigurationService,
         @inject(IApplicationShell) private applicationShell: IApplicationShell,
         @inject(IDisposableRegistry) private disposables: IDisposableRegistry,
-        @inject(ICommandManager) private commandManager: ICommandManager
+        @inject(ICommandManager) private commandManager: ICommandManager,
+        @inject(IFileSystem) private fileSystem: IFileSystem
     ) {
         // Disable gather if we're not running on insiders.
         this._enabled =
@@ -33,28 +37,12 @@ export class GatherProvider implements IGatherProvider {
                 ? true
                 : false;
 
-        if (this._enabled) {
-            try {
-                // tslint:disable-next-line: no-require-imports
-                const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis');
-
-                if (ppa) {
-                    this.dataflowAnalyzer = new ppa.DataflowAnalyzer();
-                    this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer);
-
-                    this.disposables.push(
-                        this.configService.getSettings(undefined).onDidChange((e) => this.updateEnableGather(e))
-                    );
-                }
-            } catch (ex) {
-                traceInfo(
-                    'Gathering tools could not be activated. Indicates build of VSIX could not find @msrvida/python-program-analysis'
-                );
-            }
-        }
+        this.initPromise = this.init();
     }
 
-    public logExecution(vscCell: IVscCell): void {
+    public async logExecution(vscCell: IVscCell): Promise<void> {
+        await this.initPromise;
+
         const gatherCell = convertVscToGatherCell(vscCell);
 
         if (gatherCell) {
@@ -65,6 +53,8 @@ export class GatherProvider implements IGatherProvider {
     }
 
     public async resetLog(): Promise<void> {
+        await this.initPromise;
+
         if (this._executionSlicer) {
             this._executionSlicer.reset();
         }
@@ -90,11 +80,7 @@ export class GatherProvider implements IGatherProvider {
 
         // Call internal slice method
         const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId);
-        const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker);
-
-        // Add a comment at the top of the file explaining what gather does
-        const descriptor = localize.DataScience.gatheredScriptDescription();
-        return descriptor.concat(program);
+        return slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker);
     }
 
     public get executionSlicer() {
@@ -124,6 +110,50 @@ export class GatherProvider implements IGatherProvider {
             }
         }
     }
+
+    private async init(): Promise<void> {
+        if (this._enabled) {
+            try {
+                // tslint:disable-next-line: no-require-imports
+                const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis');
+
+                if (ppa) {
+                    // If the __builtins__ specs are not available for gather, then no specs have been found. Look in a specific location relative
+                    // to the extension.
+                    if (!ppa.getSpecs()) {
+                        const defaultSpecFolder = path.join(__dirname, 'gatherSpecs');
+                        if (await this.fileSystem.directoryExists(defaultSpecFolder)) {
+                            ppa.setSpecFolder(defaultSpecFolder);
+                        }
+                    }
+
+                    // Check to see if any additional specs can be found.
+                    const additionalSpecPath = this.configService.getSettings().datascience.gatherSpecPath;
+                    if (additionalSpecPath && (await this.fileSystem.directoryExists(additionalSpecPath))) {
+                        ppa.addSpecFolder(additionalSpecPath);
+                    } else {
+                        traceInfo(`Gather: additional spec folder ${additionalSpecPath} but not found.`);
+                    }
+
+                    // Only continue to initialize gather if we were successful in finding SOME specs.
+                    if (ppa.getSpecs()) {
+                        this.dataflowAnalyzer = new ppa.DataflowAnalyzer();
+                        this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer);
+
+                        this.disposables.push(
+                            this.configService.getSettings(undefined).onDidChange((e) => this.updateEnableGather(e))
+                        );
+                    } else {
+                        this._enabled = false;
+                        traceInfo("Gather couldn't find any package specs. Disabling.");
+                    }
+                }
+            } catch (ex) {
+                this._enabled = false;
+                traceInfo(`Gathering tools could't be activated. ${ex.message}`);
+            }
+        }
+    }
 }
 
 /**