Implement IJupyterVariables for the debugger (microsoft#11543)

* Preliminary getting variables from the debugger

* Implement updating as stepping

* Get debugger and regular kernel to use same scripts

* Put back old code and create experiment to enable/disable this

* Implement experiment and begin on test for debugger

* Finish fixing tests

* Add news entry

* Update comment

* Code review feedback

* Fix crash during functional test on new setting

* Fix the ipython unit tests

* Fix formatting

Author: rchiodo

.sonarcloud.properties

@@ -1,4 +1,4 @@
 sonar.sources=src/client,src/datascience-ui
 sonar.tests=src/test
 sonar.cfamily.build-wrapper-output.bypass=true
-sonar.cpd.exclusions=src/datascience-ui/**/redux/actions.ts,src/client/**/raw-kernel/rawKernel.ts
+sonar.cpd.exclusions=src/datascience-ui/**/redux/actions.ts,src/client/**/raw-kernel/rawKernel.ts,src/client/datascience/jupyter/*ariable*.ts

.vscode/launch.json

@@ -26,7 +26,9 @@
                 // Enable this to try out new experiments locally
                 "XVSC_PYTHON_LOAD_EXPERIMENTS_FROM_FILE" : "1",
                 // Enable this to log telemetry to the output during debugging
-                "XVSC_PYTHON_LOG_TELEMETRY": "1"
+                "XVSC_PYTHON_LOG_TELEMETRY": "1",
+                // Enable this to log debugger output. Directory must exist ahead of time
+                "XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output_Ex"
             }
         },
         {
@@ -275,7 +277,9 @@
                 // Remove 'X' to turn on all logging in the debug output
                 "XVSC_PYTHON_FORCE_LOGGING": "1",
                 // Remove `X` prefix and update path to test with real python interpreter (for DS functional tests).
-                "XCI_PYTHON_PATH": "<Python Path>"
+                "XCI_PYTHON_PATH": "<Python Path>",
+                // Remove 'X' prefix to dump output for debugger. Directory has to exist prior to launch
+                "XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output"
             },
             "outFiles": [
                 "${workspaceFolder}/out/**/*.js"

experiments.json

@@ -172,5 +172,18 @@
         "salt": "DeprecatePythonPath",
         "min": 100,
         "max": 100
+    },
+    {
+        "name": "RunByLine - control",
+        "salt": "RunByLine",
+        "min": 0,
+        "max": 100
+    },
+    {
+        "name": "RunByLine - experiment",
+        "salt": "RunByLine",
+        "max": 0,
+        "min": 0
     }
+
 ]

news/3 Code Health/11542.md

@@ -0,0 +1 @@
+Implement an IJupyterVariables provider for the debugger.

package.json

@@ -1604,6 +1604,7 @@
                             "CollectLSRequestTiming - experiment",
                             "CollectNodeLSRequestTiming - experiment",
                             "DeprecatePythonPath - experiment",
+                            "RunByLine - experiment",
                             "All"
                         ]
                     },
@@ -1628,6 +1629,7 @@
                             "CollectLSRequestTiming - experiment",
                             "CollectNodeLSRequestTiming - experiment",
                             "DeprecatePythonPath - experiment",
+                            "RunByLine - experiment",
                             "All"
                         ]
                     },

pythonFiles/tests/ipython/scripts.py

@@ -43,6 +43,12 @@ def execute_script(file, replace_dict=dict([])):
     return result.success
 
 
+def execute_code(code):
+    # Execute this script as a cell
+    result = get_ipython().run_cell(code)
+    return result
+
+
 def get_variables(capsys):
     path = os.path.dirname(os.path.abspath(__file__))
     file = os.path.abspath(os.path.join(path, "./getJupyterVariableList.py"))
@@ -74,35 +80,41 @@ def get_variable_value(variables, name, capsys):
 def get_data_frame_info(variables, name, capsys):
     varJson = find_variable_json(variables, name)
     path = os.path.dirname(os.path.abspath(__file__))
-    file = os.path.abspath(
-        os.path.join(
-            path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py"
-        )
+    syspath = os.path.abspath(
+        os.path.join(path, "../../vscode_datascience_helpers/dataframes")
     )
-    keys = dict([("_VSCode_JupyterTestValue", json.dumps(varJson))])
-    if execute_script(file, keys):
+    syscode = 'import sys\nsys.path.append("{0}")'.format(syspath.replace("\\", "\\\\"))
+    importcode = "import vscodeGetDataFrameInfo\nprint(vscodeGetDataFrameInfo._VSCODE_getDataFrameInfo({0}))".format(
+        name
+    )
+    result = execute_code(syscode)
+    if not result.success:
+        result.raise_error()
+    result = execute_code(importcode)
+    if result.success:
         read_out = capsys.readouterr()
-        return json.loads(read_out.out)
+        info = json.loads(read_out.out[0:-1])
+        varJson.update(info)
+        return varJson
     else:
-        raise Exception("Get dataframe info failed.")
+        result.raise_error()
 
 
 def get_data_frame_rows(varJson, start, end, capsys):
     path = os.path.dirname(os.path.abspath(__file__))
-    file = os.path.abspath(
-        os.path.join(
-            path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameRows.py"
-        )
+    syspath = os.path.abspath(
+        os.path.join(path, "../../vscode_datascience_helpers/dataframes")
     )
-    keys = dict(
-        [
-            ("_VSCode_JupyterTestValue", json.dumps(varJson)),
-            ("_VSCode_JupyterStartRow", str(start)),
-            ("_VSCode_JupyterEndRow", str(end)),
-        ]
+    syscode = 'import sys\nsys.path.append("{0}")'.format(syspath.replace("\\", "\\\\"))
+    importcode = "import vscodeGetDataFrameRows\nprint(vscodeGetDataFrameRows._VSCODE_getDataFrameRows({0}, {1}, {2}))".format(
+        varJson["name"], start, end
     )
-    if execute_script(file, keys):
+    result = execute_code(syscode)
+    if not result.success:
+        result.raise_error()
+    result = execute_code(importcode)
+    if result.success:
         read_out = capsys.readouterr()
-        return json.loads(read_out.out)
+        return json.loads(read_out.out[0:-1])
     else:
-        raise Exception("Getting dataframe rows failed.")
+        result.raise_error()

pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py

@@ -0,0 +1,37 @@
+import pandas as _VSCODE_pd
+import builtins as _VSCODE_builtins
+
+# Function that converts the var passed in into a pandas data frame if possible
+def _VSCODE_convertToDataFrame(df):
+    if isinstance(df, list):
+        df = _VSCODE_pd.DataFrame(df)
+    elif isinstance(df, _VSCODE_pd.Series):
+        df = _VSCODE_pd.Series.to_frame(df)
+    elif isinstance(df, dict):
+        df = _VSCODE_pd.Series(df)
+        df = _VSCODE_pd.Series.to_frame(df)
+    elif hasattr(df, "toPandas"):
+        df = df.toPandas()
+    else:
+        try:
+            temp = _VSCODE_pd.DataFrame(df)
+            df = temp
+        except:
+            pass
+    return df
+
+
+# Function to compute row count for a value
+def _VSCODE_getRowCount(var):
+    if hasattr(var, "shape"):
+        try:
+            # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
+            if isinstance(var.shape, tuple):
+                return var.shape[0]
+        except TypeError:
+            return 0
+    elif hasattr(var, "__len__"):
+        try:
+            return _VSCODE_builtins.len(var)
+        except TypeError:
+            return 0

pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py

@@ -0,0 +1,53 @@
+# Query Jupyter server for the info about a dataframe
+import json as _VSCODE_json
+import pandas as _VSCODE_pd
+import pandas.io.json as _VSCODE_pd_json
+import builtins as _VSCODE_builtins
+import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
+
+# Function to do our work. It will return the object
+def _VSCODE_getDataFrameInfo(df):
+    df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df)
+    rowCount = _VSCODE_dataFrameHelpers._VSCODE_getRowCount(df)
+
+    # If any rows, use pandas json to convert a single row to json. Extract
+    # the column names and types from the json so we match what we'll fetch when
+    # we ask for all of the rows
+    if rowCount:
+        try:
+            row = df.iloc[0:1]
+            json_row = _VSCODE_pd_json.to_json(None, row, date_format="iso")
+            columnNames = list(_VSCODE_json.loads(json_row))
+        except:
+            columnNames = list(df)
+    else:
+        columnNames = list(df)
+
+    # Compute the index column. It may have been renamed
+    indexColumn = df.index.name if df.index.name else "index"
+    columnTypes = _VSCODE_builtins.list(df.dtypes)
+
+    # Make sure the index column exists
+    if indexColumn not in columnNames:
+        columnNames.insert(0, indexColumn)
+        columnTypes.insert(0, "int64")
+
+    # Then loop and generate our output json
+    columns = []
+    for n in _VSCODE_builtins.range(0, _VSCODE_builtins.len(columnNames)):
+        column_type = columnTypes[n]
+        column_name = str(columnNames[n])
+        colobj = {}
+        colobj["key"] = column_name
+        colobj["name"] = column_name
+        colobj["type"] = str(column_type)
+        columns.append(colobj)
+
+    # Save this in our target
+    target = {}
+    target["columns"] = columns
+    target["indexColumn"] = indexColumn
+    target["rowCount"] = rowCount
+
+    # return our json object as a string
+    return _VSCODE_json.dumps(target)

pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py

@@ -0,0 +1,11 @@
+# Query for the rows of a data frame
+import pandas.io.json as _VSCODE_pd_json
+import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
+
+# Function to retrieve a set of rows for a data frame
+def _VSCODE_getDataFrameRows(df, start, end):
+    df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df)
+
+    # Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON
+    rows = df.iloc[start:end]
+    return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso")

src/client/common/experimentGroups.ts

@@ -48,6 +48,12 @@ export enum LocalZMQKernel {
     experiment = 'LocalZMQKernel - experiment'
 }
 
+// Experiment for supporting run by line in data science notebooks
+export enum RunByLine {
+    control = 'RunByLine - control',
+    experiment = 'RunByLine - experiment'
+}
+
 /**
  * Experiment to check whether to to use a terminal to generate the environment variables of activated environments.
  *

src/client/common/process/internal/scripts/vscode_datascience_helpers.ts

@@ -6,24 +6,6 @@ import { _ISOLATED as ISOLATED, _SCRIPTS_DIR } from './index';
 
 const SCRIPTS_DIR = path.join(_SCRIPTS_DIR, 'vscode_datascience_helpers');
 
-//============================
-// getJupyterVariableDataFrameInfo.py
-
-export function getJupyterVariableDataFrameInfo(): string[] {
-    const script = path.join(SCRIPTS_DIR, 'getJupyterVariableDataFrameInfo.py');
-    // There is no script-specific output to parse, so we do not return a function.
-    return [ISOLATED, script];
-}
-
-//============================
-// getJupyterVariableDataFrameRows.py
-
-export function getJupyterVariableDataFrameRows(): string[] {
-    const script = path.join(SCRIPTS_DIR, 'getJupyterVariableDataFrameRows.py');
-    // There is no script-specific output to parse, so we do not return a function.
-    return [ISOLATED, script];
-}
-
 //============================
 // getServerInfo.py
 

src/client/datascience/constants.ts

@@ -2,7 +2,8 @@
 // Licensed under the MIT License.
 'use strict';
 
-import { PYTHON_LANGUAGE } from '../common/constants';
+import * as path from 'path';
+import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../common/constants';
 import { IS_WINDOWS } from '../common/platform/constants';
 import { IVariableQuery } from '../common/types';
 
@@ -374,6 +375,17 @@ export namespace Settings {
     };
 }
 
+export namespace DataFrameLoading {
+    export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes');
+    export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`;
+    export const DataFrameInfoImportName = '_VSCODE_InfoImport';
+    export const DataFrameInfoImport = `import vscodeGetDataFrameInfo as ${DataFrameInfoImportName}`;
+    export const DataFrameInfoFunc = `${DataFrameInfoImportName}._VSCODE_getDataFrameInfo`;
+    export const DataFrameRowImportName = '_VSCODE_RowImport';
+    export const DataFrameRowImport = `import vscodeGetDataFrameRows as ${DataFrameRowImportName}`;
+    export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`;
+}
+
 export namespace Identifiers {
     export const EmptyFileName = '2DB9B899-6519-4E1B-88B0-FA728A274115';
     export const GeneratedThemeName = 'ipython-theme'; // This needs to be all lower class and a valid class name.
@@ -387,6 +399,10 @@ export namespace Identifiers {
     export const InteractiveWindowIdentityScheme = 'history';
     export const DefaultCodeCellMarker = '# %%';
     export const DefaultCommTarget = 'jupyter.widget';
+    export const ALL_VARIABLES = 'ALL_VARIABLES';
+    export const OLD_VARIABLES = 'OLD_VARIABLES';
+    export const KERNEL_VARIABLES = 'KERNEL_VARIABLES';
+    export const DEBUGGER_VARIABLES = 'DEBUGGER_VARIABLES';
 }
 
 export namespace CodeSnippits {

src/client/datascience/data-viewing/dataViewer.ts

@@ -3,7 +3,7 @@
 'use strict';
 import '../../common/extensions';
 
-import { inject, injectable } from 'inversify';
+import { inject, injectable, named } from 'inversify';
 import * as path from 'path';
 import { ViewColumn } from 'vscode';
 
@@ -16,7 +16,7 @@ import * as localize from '../../common/utils/localize';
 import { noop } from '../../common/utils/misc';
 import { StopWatch } from '../../common/utils/stopWatch';
 import { sendTelemetryEvent } from '../../telemetry';
-import { HelpLinks, Telemetry } from '../constants';
+import { HelpLinks, Identifiers, Telemetry } from '../constants';
 import { JupyterDataRateLimitError } from '../jupyter/jupyterDataRateLimitError';
 import { ICodeCssGenerator, IDataViewer, IJupyterVariable, IJupyterVariables, INotebook, IThemeFinder } from '../types';
 import { WebViewHost } from '../webViewHost';
@@ -37,7 +37,7 @@ export class DataViewer extends WebViewHost<IDataViewerMapping> implements IData
         @inject(ICodeCssGenerator) cssGenerator: ICodeCssGenerator,
         @inject(IThemeFinder) themeFinder: IThemeFinder,
         @inject(IWorkspaceService) workspaceService: IWorkspaceService,
-        @inject(IJupyterVariables) private variableManager: IJupyterVariables,
+        @inject(IJupyterVariables) @named(Identifiers.ALL_VARIABLES) private variableManager: IJupyterVariables,
         @inject(IApplicationShell) private applicationShell: IApplicationShell,
         @inject(IExperimentsManager) experimentsManager: IExperimentsManager,
         @inject(UseCustomEditorApi) useCustomEditorApi: boolean
@@ -54,7 +54,8 @@ export class DataViewer extends WebViewHost<IDataViewerMapping> implements IData
             localize.DataScience.dataExplorerTitle(),
             ViewColumn.One,
             experimentsManager.inExperiment(WebHostNotebook.experiment),
-            useCustomEditorApi
+            useCustomEditorApi,
+            false
         );
 
         // Load the web panel using our current directory as we don't expect to load any other files

src/client/datascience/interactive-common/interactiveBase.ts

@@ -35,7 +35,7 @@ import {
 } from '../../common/application/types';
 import { CancellationError } from '../../common/cancellation';
 import { EXTENSION_ROOT_DIR, isTestExecution, PYTHON_LANGUAGE } from '../../common/constants';
-import { WebHostNotebook } from '../../common/experimentGroups';
+import { RunByLine, WebHostNotebook } from '../../common/experimentGroups';
 import { traceError, traceInfo, traceWarning } from '../../common/logger';
 import { IFileSystem } from '../../common/platform/types';
 import { IConfigurationService, IDisposableRegistry, IExperimentsManager } from '../../common/types';
@@ -164,7 +164,8 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
             title,
             viewColumn,
             experimentsManager.inExperiment(WebHostNotebook.experiment),
-            useCustomEditorApi
+            useCustomEditorApi,
+            experimentsManager.inExperiment(RunByLine.experiment)
         );
 
         // Create our unique id. We use this to skip messages we send to other interactive windows
@@ -197,6 +198,9 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
 
         // When a server starts, make sure we create a notebook if the server matches
         jupyterExecution.serverStarted(this.checkForNotebookProviderConnection.bind(this));
+
+        // When the variable service requests a refresh, refresh our variable list
+        this.disposables.push(this.jupyterVariables.refreshRequired(this.refreshVariables.bind(this)));
     }
 
     public async show(): Promise<void> {
@@ -1147,6 +1151,10 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
         }
     }
 
+    private refreshVariables() {
+        this.postMessage(InteractiveWindowMessages.ForceVariableRefresh).ignoreErrors();
+    }
+
     private async checkForNotebookProviderConnection(): Promise<void> {
         // Check to see if we are already connected to our provider
         const providerConnection = await this.notebookProvider.connect({ getOnly: true });