Skip to content

Commit 6b6b959

Browse files
rchiodorchiodo
authored
Rchiodo/kernel telemetry (#10115)
* Add duration to select local/remote kernel * Add notebook language telemetry * Add news entries * #9883 telemetry * News entry * Another spot for kernel spec failure * Add telemetry on product install * Fix install telemetry * Undo launch.json change * Handle other cases * Better way to handle case * Wrong event for jupyter install * Fix unit tests
1 parent 29f8f4c commit 6b6b959

File tree

16 files changed

+268
-62
lines changed

16 files changed

+268
-62
lines changed

‎news/3 Code Health/10049.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Change select kernel telemetry to track duration till quick pick appears.

‎news/3 Code Health/9819.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add telemetry to track notebook languages

‎news/3 Code Health/9883.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Telemetry around kernels not working and installs not working.

‎src/client/common/installer/productInstaller.ts

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as os from 'os';
55
import { CancellationToken, OutputChannel, Uri } from 'vscode';
66
import '../../common/extensions';
77
import * as localize from '../../common/utils/localize';
8+
import { Telemetry } from '../../datascience/constants';
89
import { IInterpreterService } from '../../interpreter/contracts';
910
import { IServiceContainer } from '../../ioc/types';
1011
import { LinterId } from '../../linters/types';
@@ -27,6 +28,7 @@ import {
2728
ProductType
2829
} from '../types';
2930
import { isResource } from '../utils/misc';
31+
import { StopWatch } from '../utils/stopWatch';
3032
import { ProductNames } from './productNames';
3133
import { IInstallationChannelManager, InterpreterUri, IProductPathService, IProductService } from './types';
3234

@@ -355,7 +357,22 @@ export class DataScienceInstaller extends BaseInstaller {
355357
'Yes',
356358
'No'
357359
);
358-
return item === 'Yes' ? this.install(product, resource, cancel) : InstallerResponse.Ignore;
360+
if (item === 'Yes') {
361+
const stopWatch = new StopWatch();
362+
try {
363+
const response = await this.install(product, resource, cancel);
364+
const event =
365+
product === Product.jupyter ? Telemetry.UserInstalledJupyter : Telemetry.UserInstalledModule;
366+
sendTelemetryEvent(event, stopWatch.elapsedTime, { product: productName });
367+
return response;
368+
} catch (e) {
369+
if (product === Product.jupyter) {
370+
sendTelemetryEvent(Telemetry.JupyterInstallFailed);
371+
}
372+
throw e;
373+
}
374+
}
375+
return InstallerResponse.Ignore;
359376
}
360377
}
361378

‎src/client/datascience/constants.ts

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,21 @@ export const JUPYTER_OUTPUT_CHANNEL = 'JUPYTER_OUTPUT_CHANNEL';
1414
// Python Module to be used when instantiating the Python Daemon.
1515
export const PythonDaemonModule = 'datascience.jupyter_daemon';
1616

17+
// List of 'language' names that we know about. All should be lower case as that's how we compare.
18+
export const KnownNotebookLanguages: string[] = [
19+
'python',
20+
'r',
21+
'julia',
22+
'c++',
23+
'c#',
24+
'f#',
25+
'scala',
26+
'haskell',
27+
'bash',
28+
'cling',
29+
'sas'
30+
];
31+
1732
export namespace Commands {
1833
export const RunAllCells = 'python.datascience.runallcells';
1934
export const RunAllCellsAbove = 'python.datascience.runallcellsabove';
@@ -153,7 +168,7 @@ export enum Telemetry {
153168
CollapseAll = 'DATASCIENCE.COLLAPSE_ALL',
154169
SelectJupyterURI = 'DATASCIENCE.SELECT_JUPYTER_URI',
155170
SelectLocalJupyterKernel = 'DATASCIENCE.SELECT_LOCAL_JUPYTER_KERNEL',
156-
SelectRemoteJupyuterKernel = 'DATASCIENCE.SELECT_REMOTE_JUPYTER_KERNEL',
171+
SelectRemoteJupyterKernel = 'DATASCIENCE.SELECT_REMOTE_JUPYTER_KERNEL',
157172
SetJupyterURIToLocal = 'DATASCIENCE.SET_JUPYTER_URI_LOCAL',
158173
SetJupyterURIToUserSpecified = 'DATASCIENCE.SET_JUPYTER_URI_USER_SPECIFIED',
159174
Interrupt = 'DATASCIENCE.INTERRUPT',
@@ -246,6 +261,12 @@ export enum Telemetry {
246261
FindKernelForLocalConnection = 'DS_INTERNAL.FIND_KERNEL_FOR_LOCAL_CONNECTION',
247262
CompletionTimeFromLS = 'DS_INTERNAL.COMPLETION_TIME_FROM_LS',
248263
CompletionTimeFromJupyter = 'DS_INTERNAL.COMPLETION_TIME_FROM_JUPYTER',
264+
NotebookLanguage = 'DATASCIENCE.NOTEBOOK_LANGUAGE',
265+
KernelSpecNotFound = 'DS_INTERNAL.KERNEL_SPEC_NOT_FOUND',
266+
KernelRegisterFailed = 'DS_INTERNAL.KERNEL_REGISTER_FAILED',
267+
KernelEnumeration = 'DS_INTERNAL.KERNEL_ENUMERATION',
268+
JupyterInstallFailed = 'DS_INTERNAL.JUPYTER_INSTALL_FAILED',
269+
UserInstalledModule = 'DATASCIENCE.USER_INSTALLED_MODULE',
249270
JupyterCommandLineNonDefault = 'DS_INTERNAL.JUPYTER_CUSTOM_COMMAND_LINE'
250271
}
251272

‎src/client/datascience/interactive-ipynb/nativeEditor.ts

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import { captureTelemetry, sendTelemetryEvent } from '../../telemetry';
4343
import {
4444
EditorContexts,
4545
Identifiers,
46+
KnownNotebookLanguages,
4647
NativeKeyboardCommandTelemetryLookup,
4748
NativeMouseCommandTelemetryLookup,
4849
Telemetry
@@ -432,7 +433,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor {
432433
}
433434
}
434435

435-
@captureTelemetry(Telemetry.ExecuteNativeCell, undefined, false)
436+
@captureTelemetry(Telemetry.ExecuteNativeCell, undefined, true)
436437
// tslint:disable-next-line:no-any
437438
protected async reexecuteCell(info: ISubmitNewCell): Promise<void> {
438439
try {
@@ -627,6 +628,9 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor {
627628
// Then save the contents. We'll stick our cells back into this format when we save
628629
if (json) {
629630
this.notebookJson = json;
631+
632+
// Log language or kernel telemetry
633+
this.sendLanguageTelemetry(this.notebookJson);
630634
}
631635
this.contentsLoadedPromise.resolve();
632636

@@ -650,6 +654,27 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor {
650654
);
651655
}
652656

657+
private sendLanguageTelemetry(notebookJson: Partial<nbformat.INotebookContent>) {
658+
try {
659+
// See if we have a language
660+
let language = '';
661+
if (notebookJson.metadata?.language_info?.name) {
662+
language = notebookJson.metadata?.language_info?.name;
663+
} else if (notebookJson.metadata?.kernelspec?.language) {
664+
language = notebookJson.metadata?.kernelspec?.language.toString();
665+
}
666+
if (language && !KnownNotebookLanguages.includes(language.toLowerCase())) {
667+
language = 'unknown';
668+
}
669+
if (language) {
670+
sendTelemetryEvent(Telemetry.NotebookLanguage, undefined, { language });
671+
}
672+
} catch {
673+
// If this fails, doesn't really matter
674+
noop();
675+
}
676+
}
677+
653678
private async loadCells(cells: ICell[], forceDirty: boolean): Promise<void> {
654679
// Make sure cells have at least 1
655680
if (cells.length === 0) {

‎src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterDependencyService.ts

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { ProductNames } from '../../../common/installer/productNames';
99
import { IInstallationChannelManager } from '../../../common/installer/types';
1010
import { Product } from '../../../common/types';
1111
import { DataScience } from '../../../common/utils/localize';
12+
import { StopWatch } from '../../../common/utils/stopWatch';
1213
import { sendTelemetryEvent } from '../../../telemetry';
1314
import { Telemetry } from '../../constants';
1415
import { IJupyterInterpreterDependencyManager } from '../../types';
@@ -37,20 +38,28 @@ export class JupyterCommandInterpreterDependencyService implements IJupyterInter
3738
// If Conda is available, always pick it as the user must have a Conda Environment
3839
const installer = installers.find(ins => ins.name === 'Conda');
3940
const product = ProductNames.get(Product.jupyter);
41+
const stopWatch = new StopWatch();
4042

4143
if (installer && product) {
42-
sendTelemetryEvent(Telemetry.UserInstalledJupyter);
4344
installer
4445
.installModule(product)
45-
.catch(e =>
46-
this.applicationShell.showErrorMessage(e.message, DataScience.pythonInteractiveHelpLink())
47-
);
46+
.then(() => {
47+
sendTelemetryEvent(Telemetry.UserInstalledJupyter, stopWatch.elapsedTime);
48+
})
49+
.catch(e => {
50+
sendTelemetryEvent(Telemetry.JupyterInstallFailed, undefined, { product });
51+
this.applicationShell.showErrorMessage(e.message, DataScience.pythonInteractiveHelpLink());
52+
});
4853
} else if (installers[0] && product) {
4954
installers[0]
5055
.installModule(product)
51-
.catch(e =>
52-
this.applicationShell.showErrorMessage(e.message, DataScience.pythonInteractiveHelpLink())
53-
);
56+
.then(() => {
57+
sendTelemetryEvent(Telemetry.UserInstalledJupyter, stopWatch.elapsedTime);
58+
})
59+
.catch(e => {
60+
sendTelemetryEvent(Telemetry.JupyterInstallFailed, undefined, { product });
61+
this.applicationShell.showErrorMessage(e.message, DataScience.pythonInteractiveHelpLink());
62+
});
5463
}
5564
}
5665
} else if (response === DataScience.notebookCheckForImportNo()) {

‎src/client/datascience/jupyter/interpreter/jupyterInterpreterDependencyService.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,10 @@ export class JupyterInterpreterDependencyService {
303303
return execService
304304
.execModule('jupyter', ['kernelspec', '--version'], { throwOnStdErr: true })
305305
.then(() => true)
306-
.catch(() => false);
306+
.catch(() => {
307+
sendTelemetryEvent(Telemetry.KernelSpecNotFound);
308+
return false;
309+
});
307310
}
308311

309312
/**

‎src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ import { DataScience } from '../../../common/utils/localize';
1515
import { noop } from '../../../common/utils/misc';
1616
import { EXTENSION_ROOT_DIR } from '../../../constants';
1717
import { IInterpreterService, PythonInterpreter } from '../../../interpreter/contracts';
18-
import { JUPYTER_OUTPUT_CHANNEL, PythonDaemonModule } from '../../constants';
18+
import { sendTelemetryEvent } from '../../../telemetry';
19+
import { JUPYTER_OUTPUT_CHANNEL, PythonDaemonModule, Telemetry } from '../../constants';
1920
import { IJupyterInterpreterDependencyManager, IJupyterSubCommandExecutionService } from '../../types';
2021
import { JupyterServerInfo } from '../jupyterConnection';
2122
import { JupyterInstallError } from '../jupyterInstallError';
@@ -192,6 +193,7 @@ export class JupyterInterpreterSubCommandExecutionService
192193
.execModule('jupyter', ['kernelspec', 'list', '--json'], spawnOptions)
193194
.then(output => output.stdout)
194195
.catch(daemonEx => {
196+
sendTelemetryEvent(Telemetry.KernelSpecNotFound);
195197
traceError('Failed to list kernels from daemon', daemonEx);
196198
return '';
197199
});

‎src/client/datascience/jupyter/jupyterExecution.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ export class JupyterExecutionBase implements IJupyterExecution {
221221
>(IJupyterSessionManagerFactory);
222222
const sessionManager = await sessionManagerFactory.create(connection);
223223
const kernelInterpreter = await this.kernelSelector.selectLocalKernel(
224+
new StopWatch(),
224225
sessionManager,
225226
cancelToken,
226227
launchInfo.kernelSpec

‎src/client/datascience/jupyter/kernels/kernelSelector.ts

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import { noop } from '../../../common/utils/misc';
1515
import { StopWatch } from '../../../common/utils/stopWatch';
1616
import { IInterpreterService, PythonInterpreter } from '../../../interpreter/contracts';
1717
import { IEventNamePropertyMapping, sendTelemetryEvent } from '../../../telemetry';
18-
import { Telemetry } from '../../constants';
18+
import { KnownNotebookLanguages, Telemetry } from '../../constants';
1919
import { reportAction } from '../../progress/decorator';
2020
import { ReportableAction } from '../../progress/types';
2121
import { IJupyterKernelSpec, IJupyterSessionManager } from '../../types';
@@ -90,13 +90,21 @@ export class KernelSelector {
9090
* @memberof KernelSelector
9191
*/
9292
public async selectRemoteKernel(
93+
stopWatch: StopWatch,
9394
session: IJupyterSessionManager,
9495
cancelToken?: CancellationToken,
9596
currentKernel?: IJupyterKernelSpec | LiveKernelModel
9697
): Promise<KernelSpecInterpreter> {
9798
let suggestions = await this.selectionProvider.getKernelSelectionsForRemoteSession(session, cancelToken);
9899
suggestions = suggestions.filter(item => !this.kernelIdsToHide.has(item.selection.kernelModel?.id || ''));
99-
return this.selectKernel(suggestions, session, cancelToken, currentKernel);
100+
return this.selectKernel(
101+
stopWatch,
102+
Telemetry.SelectRemoteJupyterKernel,
103+
suggestions,
104+
session,
105+
cancelToken,
106+
currentKernel
107+
);
100108
}
101109
/**
102110
* Select a kernel from a local session.
@@ -107,13 +115,21 @@ export class KernelSelector {
107115
* @memberof KernelSelector
108116
*/
109117
public async selectLocalKernel(
118+
stopWatch: StopWatch,
110119
session?: IJupyterSessionManager,
111120
cancelToken?: CancellationToken,
112121
currentKernel?: IJupyterKernelSpec | LiveKernelModel
113122
): Promise<KernelSpecInterpreter> {
114123
let suggestions = await this.selectionProvider.getKernelSelectionsForLocalSession(session, cancelToken);
115124
suggestions = suggestions.filter(item => !this.kernelIdsToHide.has(item.selection.kernelModel?.id || ''));
116-
return this.selectKernel(suggestions, session, cancelToken, currentKernel);
125+
return this.selectKernel(
126+
stopWatch,
127+
Telemetry.SelectLocalJupyterKernel,
128+
suggestions,
129+
session,
130+
cancelToken,
131+
currentKernel
132+
);
117133
}
118134
/**
119135
* Gets a kernel that needs to be used with a local session.
@@ -168,7 +184,7 @@ export class KernelSelector {
168184
);
169185
} else {
170186
telemetryProps.promptedToSelect = true;
171-
selection = await this.selectLocalKernel(sessionManager, cancelToken);
187+
selection = await this.selectLocalKernel(stopWatch, sessionManager, cancelToken);
172188
}
173189
}
174190
} else {
@@ -263,6 +279,8 @@ export class KernelSelector {
263279
};
264280
}
265281
private async selectKernel(
282+
stopWatch: StopWatch,
283+
telemetryEvent: Telemetry,
266284
suggestions: IKernelSpecQuickPickItem[],
267285
session?: IJupyterSessionManager,
268286
cancelToken?: CancellationToken,
@@ -271,6 +289,7 @@ export class KernelSelector {
271289
const placeHolder =
272290
localize.DataScience.selectKernel() +
273291
(currentKernel ? ` (current: ${currentKernel.display_name || currentKernel.name})` : '');
292+
sendTelemetryEvent(telemetryEvent, stopWatch.elapsedTime);
274293
const selection = await this.applicationShell.showQuickPick(suggestions, { placeHolder }, cancelToken);
275294
if (!selection?.selection) {
276295
return {};
@@ -280,7 +299,9 @@ export class KernelSelector {
280299
sendTelemetryEvent(Telemetry.SwitchToInterpreterAsKernel);
281300
return this.useInterpreterAsKernel(selection.selection.interpreter, undefined, session, false, cancelToken);
282301
} else if (selection.selection.kernelModel) {
283-
sendTelemetryEvent(Telemetry.SwitchToExistingKernel);
302+
sendTelemetryEvent(Telemetry.SwitchToExistingKernel, undefined, {
303+
language: this.computeLanguage(selection.selection.kernelModel.language)
304+
});
284305
// tslint:disable-next-line: no-any
285306
const interpreter = selection.selection.kernelModel
286307
? await this.kernelService.findMatchingInterpreter(selection.selection.kernelModel, cancelToken)
@@ -291,7 +312,9 @@ export class KernelSelector {
291312
kernelModel: selection.selection.kernelModel
292313
};
293314
} else if (selection.selection.kernelSpec) {
294-
sendTelemetryEvent(Telemetry.SwitchToExistingKernel);
315+
sendTelemetryEvent(Telemetry.SwitchToExistingKernel, undefined, {
316+
language: this.computeLanguage(selection.selection.kernelSpec.language)
317+
});
295318
const interpreter = selection.selection.kernelSpec
296319
? await this.kernelService.findMatchingInterpreter(selection.selection.kernelSpec, cancelToken)
297320
: undefined;
@@ -348,7 +371,12 @@ export class KernelSelector {
348371
}
349372

350373
// Try an install this interpreter as a kernel.
351-
kernelSpec = await this.kernelService.registerKernel(interpreter, disableUI, cancelToken);
374+
try {
375+
kernelSpec = await this.kernelService.registerKernel(interpreter, disableUI, cancelToken);
376+
} catch (e) {
377+
sendTelemetryEvent(Telemetry.KernelRegisterFailed);
378+
throw e;
379+
}
352380

353381
// If we have a display name of a kernel that could not be found,
354382
// then notify user that we're using current interpreter instead.
@@ -368,4 +396,11 @@ export class KernelSelector {
368396

369397
return { kernelSpec, interpreter };
370398
}
399+
400+
private computeLanguage(language: string | undefined): string {
401+
if (language && KnownNotebookLanguages.includes(language.toLowerCase())) {
402+
return language;
403+
}
404+
return 'unknown';
405+
}
371406
}

‎src/client/datascience/jupyter/kernels/kernelService.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,17 @@ export class KernelService {
424424
return [];
425425
}
426426
const specs: IJupyterKernelSpec[] = await enumerator;
427-
return specs.filter(item => !!item);
427+
const result = specs.filter(item => !!item);
428+
429+
// Send telemetry on this enumeration.
430+
const anyPython = result.find(k => k.language === 'python') !== undefined;
431+
sendTelemetryEvent(Telemetry.KernelEnumeration, undefined, {
432+
count: result.length,
433+
isPython: anyPython,
434+
source: sessionManager ? 'connection' : 'cli'
435+
});
436+
437+
return result;
428438
}
429439
/**
430440
* Not all characters are allowed in a kernel name.

0 commit comments

Comments
 (0)