Skip to content

Commit

Permalink
Use deno instead of Julia & .NET (#14943)
Browse files Browse the repository at this point in the history
* Use deno instead of Julia

* deno version

* Install deno kernel

* Fixes

* Log path to deno kernel

* Remove unwanted files

* No need of postinstall for smoke tests

* Fix tests
  • Loading branch information
DonJayamanne authored Dec 21, 2023
1 parent 941cef3 commit 0ee7acf
Show file tree
Hide file tree
Showing 7 changed files with 137 additions and 209 deletions.
33 changes: 6 additions & 27 deletions .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ env:
NODE_VERSION: 18.15.0
NPM_VERSION: 9.5.0
PYTHON_VERSION: 3.8
JULIA_VERSION: 1.5.2
DENO_VERSION: '~1.37'
MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter). Also enables a reporter which exits the process running the tests if it haven't already.
CACHE_NPM_DEPS: cache-npm
CACHE_OUT_DIRECTORY: cache-out-directory
Expand All @@ -47,7 +47,6 @@ env:
DISABLE_INSIDERS_EXTENSION: 1 # Disable prompts to install pre-release in tests (else it blocks activation of extension).
VSC_JUPYTER_INSTRUMENT_CODE_FOR_COVERAGE: true
VSC_JUPYTER_LOG_KERNEL_OUTPUT: true
DOTNET_VERSION: 8.0.x

jobs:
# Make sure to cancel previous runs on a push
Expand Down Expand Up @@ -602,35 +601,15 @@ jobs:

# Used by tests for non-python kernels.
# Test are enabled via env variable `VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST`
- name: Install Julia
- name: Install Deno
if: matrix.os != 'windows-latest' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@widgets' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@iw' && matrix.tags != '@webview|@export|@lsp|@variableViewer'
uses: julia-actions/setup-julia@v1
uses: denoland/setup-deno@v1
with:
version: ${{env.JULIA_VERSION}}
deno-version: ${{ env.DENO_VERSION}}

- name: Install Julia Kernel
- name: Install Deno Kernel
if: matrix.os != 'windows-latest' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@widgets' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@iw' && matrix.tags != '@webview|@export|@lsp|@variableViewer'
shell: bash
run: |
julia -e '
using Pkg
Pkg.add("IJulia")'
- name: Install Dot.net
if: matrix.os != 'windows-latest' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@widgets' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@iw' && matrix.tags != '@webview|@export|@lsp|@variableViewer'
uses: actions/[email protected]
with:
dotnet-version: ${{env.DOTNET_VERSION}}

- name: Install .NET Interactive
if: matrix.os != 'windows-latest' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@widgets' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@iw' && matrix.tags != '@webview|@export|@lsp|@variableViewer'
shell: bash -l {0}
run: dotnet tool install -g --add-source "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" Microsoft.dotnet-interactive

- name: Install .NET Kernel
if: matrix.os != 'windows-latest' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@widgets' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@iw' && matrix.tags != '@webview|@export|@lsp|@variableViewer'
shell: bash -l {0}
run: dotnet interactive jupyter install
run: npx tsx ./build/installDenoKernel.ts

- name: Create Virtual Env for Tests
uses: ./.github/actions/create-venv-for-tests
Expand Down
92 changes: 92 additions & 0 deletions build/installDenoKernel.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs-extra';
import { execSync } from 'child_process';

const winJupyterPath = path.join('AppData', 'Roaming', 'jupyter', 'kernels');
const linuxJupyterPath = path.join('.local', 'share', 'jupyter', 'kernels');
const macJupyterPath = path.join('Library', 'Jupyter', 'kernels');

enum OSType {
Unknown = 'Unknown',
Windows = 'Windows',
OSX = 'OSX',
Linux = 'Linux'
}

// Return the OS type for the given platform string.
function getOSType(platform: string = process.platform): OSType {
if (/^win/.test(platform)) {
return OSType.Windows;
} else if (/^darwin/.test(platform)) {
return OSType.OSX;
} else if (/^linux/.test(platform)) {
return OSType.Linux;
} else {
return OSType.Unknown;
}
}

// Home path depends upon OS
const homePath = os.homedir();

function getEnvironmentVariable(key: string): string | undefined {
return process.env[key];
}

function getUserHomeDir(): string {
if (getOSType() === OSType.Windows) {
return getEnvironmentVariable('USERPROFILE') || homePath;
}
const homeVar = getEnvironmentVariable('HOME') || getEnvironmentVariable('HOMEPATH') || homePath;

// Make sure if linux, it uses linux separators
return homeVar.replace(/\\/g, '/');
}

function getKernelSpecRootPath() {
switch (getOSType()) {
case OSType.Windows:
return path.join(getUserHomeDir(), winJupyterPath);
case OSType.OSX:
return path.join(getUserHomeDir(), macJupyterPath);
default:
return path.join(getUserHomeDir(), linuxJupyterPath);
}
}

function getDenoExec() {
return execSync('which deno').toString().trim();
}

function getDenoKernelSpecPath() {
return path.join(getKernelSpecRootPath(), 'deno', 'kernel.json');
}

function registerKernel() {
const denoKernelSpecPath = getDenoKernelSpecPath();
if (fs.existsSync(denoKernelSpecPath)) {
console.log(`Deno kernel already registered at ${denoKernelSpecPath}`);
return;
}

fs.mkdirpSync(path.dirname(denoKernelSpecPath));
fs.writeFileSync(
denoKernelSpecPath,
JSON.stringify(
{
argv: [getDenoExec(), '--unstable', 'jupyter', '--kernel', '--conn', '{connection_file}'],
display_name: 'Deno',
language: 'typescript'
},
null,
4
)
);
console.log(`Deno kernel registered at ${denoKernelSpecPath}`);
}

registerKernel();
118 changes: 11 additions & 107 deletions src/test/datascience/notebook/nonPythonKernels.vscode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,12 @@ import assert from 'assert';
import * as sinon from 'sinon';
import { Uri } from 'vscode';
import { IKernelProvider } from '../../../kernels/types';
import { IControllerRegistration } from '../../../notebooks/controllers/types';
import { PythonExtensionChecker } from '../../../platform/api/pythonApi';
import { IPythonExtensionChecker } from '../../../platform/api/types';
import { IDisposable } from '../../../platform/common/types';
import { traceInfo } from '../../../platform/logging';
import * as path from '../../../platform/vscode-path/path';
import { IExtensionTestApi, waitForCondition } from '../../common.node';
import { EXTENSION_ROOT_DIR_FOR_TESTS, IS_NON_RAW_NATIVE_TEST, IS_REMOTE_NATIVE_TEST } from '../../constants.node';
import { noop } from '../../core';
import { initialize } from '../../initialize.node';
import { ControllerPreferredService } from './controllerPreferredService';
import { TestNotebookDocument, createKernelController } from './executionHelper';
Expand All @@ -29,24 +26,16 @@ import {

/* eslint-disable @typescript-eslint/no-explicit-any, no-invalid-this */
suite('Non-Python Kernel @nonPython ', async function () {
const juliaNb = Uri.file(
path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'notebook', 'simpleJulia.ipynb')
);
const csharpNb = Uri.file(
path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'notebook', 'simpleCSharp.ipynb')
const denoNb = Uri.file(
path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'src', 'test', 'datascience', 'notebook', 'simpleDeno.ipynb')
);

let api: IExtensionTestApi;
const disposables: IDisposable[] = [];
let testJuliaNb: Uri;
let testCSharpNb: Uri;
let testDenoNb: Uri;
let controllerPreferred: ControllerPreferredService;
let kernelProvider: IKernelProvider;
let pythonChecker: IPythonExtensionChecker;
let controllerRegistration: IControllerRegistration;
this.timeout(120_000); // Julia and C# kernels can be slow
suiteSetup(async function () {
this.timeout(120_000);
api = await initialize();
verifyPromptWasNotDisplayed();
if (
Expand All @@ -60,9 +49,7 @@ suite('Non-Python Kernel @nonPython ', async function () {
sinon.restore();
verifyPromptWasNotDisplayed();
controllerPreferred = ControllerPreferredService.create(api.serviceContainer);
controllerRegistration = api.serviceContainer.get<IControllerRegistration>(IControllerRegistration);
kernelProvider = api.serviceContainer.get<IKernelProvider>(IKernelProvider);
pythonChecker = api.serviceContainer.get<IPythonExtensionChecker>(IPythonExtensionChecker);
});
function verifyPromptWasNotDisplayed() {
assert.strictEqual(
Expand All @@ -77,23 +64,22 @@ suite('Non-Python Kernel @nonPython ', async function () {
await closeNotebooks();
// Don't use same file (due to dirty handling, we might save in dirty.)
// Coz we won't save to file, hence extension will backup in dirty file and when u re-open it will open from dirty.
testJuliaNb = await createTemporaryNotebookFromFile(juliaNb, disposables);
testCSharpNb = await createTemporaryNotebookFromFile(csharpNb, disposables);
testDenoNb = await createTemporaryNotebookFromFile(denoNb, disposables);
traceInfo(`Start Test (completed) ${this.currentTest?.title}`);
});
teardown(async () => {
verifyPromptWasNotDisplayed();
await closeNotebooksAndCleanUpAfterTests(disposables);
});
// https://github.com/microsoft/vscode-jupyter/issues/10900
test('Automatically pick julia kernel when opening a Julia Notebook', async () => {
const notebook = await TestNotebookDocument.openFile(testJuliaNb);
test('Automatically pick Deno kernel when opening a Deno Notebook', async () => {
const notebook = await TestNotebookDocument.openFile(testDenoNb);
await waitForCondition(
async () => {
const preferredController = await controllerPreferred.computePreferred(notebook);
if (
preferredController.preferredConnection?.kind === 'startUsingLocalKernelSpec' &&
preferredController.preferredConnection.kernelSpec.language === 'julia'
preferredController.preferredConnection.kernelSpec.language === 'typescript'
) {
return preferredController.preferredConnection;
}
Expand All @@ -105,48 +91,14 @@ suite('Non-Python Kernel @nonPython ', async function () {
500
);
});
test('Automatically pick csharp kernel when opening a csharp notebook', async function () {
// C# Kernels can only be installed when you have Jupyter
// On CI we install Jupyter only when testing with Python extension.
if (!pythonChecker.isPythonExtensionInstalled) {
return this.skip();
}

const notebook = await TestNotebookDocument.openFile(testCSharpNb);
await waitForCondition(
async () => {
const preferredController = await controllerPreferred.computePreferred(notebook);
if (
preferredController.preferredConnection?.kind === 'startUsingLocalKernelSpec' &&
preferredController.preferredConnection.kernelSpec.language === 'C#'
) {
return preferredController.preferredConnection;
}
},
defaultNotebookTestTimeout,
`Preferred controller not found for Notebook, currently preferred ${controllerPreferred.getPreferred(
notebook
)?.connection.kind}:${controllerPreferred.getPreferred(notebook)?.connection
.id}, current controllers include ${controllerRegistration.all
.map(
(item) =>
`${item.kind}:${item.id}(${
item.kind === 'startUsingLocalKernelSpec' ? item.kernelSpec.language : ''
})`
)
.join(',')}`,
500
);
});
test('Bogus test', noop);
test('Can run a Julia notebook', async function () {
const notebook = await TestNotebookDocument.openFile(testJuliaNb);
test('Can run a Deno notebook', async function () {
const notebook = await TestNotebookDocument.openFile(testDenoNb);
const metadata = await waitForCondition(
async () => {
const preferredController = await controllerPreferred.computePreferred(notebook);
if (
preferredController.preferredConnection?.kind === 'startUsingLocalKernelSpec' &&
preferredController.preferredConnection.kernelSpec.language === 'julia'
preferredController.preferredConnection.kernelSpec.language === 'typescript'
) {
return preferredController.preferredConnection;
}
Expand All @@ -157,7 +109,7 @@ suite('Non-Python Kernel @nonPython ', async function () {
)?.connection.kind}:${controllerPreferred.getPreferred(notebook)?.connection.id}`,
500
);
const cell = await notebook.appendCodeCell('123456', 'julia');
const cell = await notebook.appendCodeCell('123456', 'typescript');
const kernel = kernelProvider.getOrCreate(notebook, {
controller: createKernelController(),
metadata,
Expand All @@ -171,52 +123,4 @@ suite('Non-Python Kernel @nonPython ', async function () {
waitForTextOutput(cell, '123456', 0, false)
]);
});
test('Can run a CSharp notebook', async function () {
// C# Kernels can only be installed when you have Jupyter
// On CI we install Jupyter only when testing with Python extension.
if (!pythonChecker.isPythonExtensionInstalled) {
return this.skip();
}

const notebook = await TestNotebookDocument.openFile(testCSharpNb);
const metadata = await waitForCondition(
async () => {
const preferredController = await controllerPreferred.computePreferred(notebook);
if (
preferredController.preferredConnection?.kind === 'startUsingLocalKernelSpec' &&
preferredController.preferredConnection.kernelSpec.language === 'C#'
) {
return preferredController.preferredConnection;
}
},
defaultNotebookTestTimeout,
`Preferred controller not found for Notebook, currently preferred ${controllerPreferred.getPreferred(
notebook
)?.connection.kind}:${controllerPreferred.getPreferred(notebook)?.connection.id}`,
500
);
const kernel = kernelProvider.getOrCreate(notebook, {
controller: createKernelController(),
metadata,
resourceUri: notebook.uri
});
const kernelExecution = kernelProvider.getKernelExecution(kernel);

const cell = notebook.cellAt(0);
// Wait till execution count changes and status is success.
await Promise.all([kernelExecution.executeCell(cell), waitForExecutionCompletedSuccessfully(cell)]);

// For some reason C# kernel sends multiple outputs.
// First output can contain `text/html` with some Jupyter UI specific stuff.
try {
traceInfo(`Cell output length ${cell.outputs.length}`);
await waitForTextOutput(cell, 'Hello', 0, false, 5_000);
} catch (ex) {
if (cell.outputs.length > 1) {
await waitForTextOutput(cell, 'Hello', 1, false);
} else {
throw ex;
}
}
});
});
30 changes: 0 additions & 30 deletions src/test/datascience/notebook/simpleCSharp.ipynb

This file was deleted.

Loading

0 comments on commit 0ee7acf

Please sign in to comment.