From e7d854ad0b337546397eacd1ca222d448d6d676f Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 27 Sep 2024 22:16:42 +0200 Subject: [PATCH] Frontend driven indexing --- package.json | 3 +- src/index.ts | 133 +++----------------------- src/quickopen.ts | 239 +++++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 10 ++ 4 files changed, 262 insertions(+), 123 deletions(-) create mode 100644 src/quickopen.ts diff --git a/package.json b/package.json index 8e35d69..6dfd65f 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "@jupyterlab/filebrowser": "^4.2.5", "@jupyterlab/services": "^7.2.5", "@jupyterlab/settingregistry": "^4.2.5", - "@jupyterlab/translation": "^4.2.5" + "@jupyterlab/translation": "^4.2.5", + "minimatch": "^10.0.1" }, "devDependencies": { "@jupyterlab/builder": "^4.0.0", diff --git a/src/index.ts b/src/index.ts index 4ff456d..4e1bb2e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,125 +3,13 @@ import { JupyterFrontEndPlugin } from '@jupyterlab/application'; import { ICommandPalette, ModalCommandPalette } from '@jupyterlab/apputils'; -import { URLExt, PathExt } from '@jupyterlab/coreutils'; +import { PathExt } from '@jupyterlab/coreutils'; import { IDocumentManager } from '@jupyterlab/docmanager'; -import { ServerConnection } from '@jupyterlab/services'; import { ISettingRegistry } from '@jupyterlab/settingregistry'; -import { FileBrowser, IDefaultFileBrowser } from '@jupyterlab/filebrowser'; +import { IDefaultFileBrowser } from '@jupyterlab/filebrowser'; import { ITranslator, nullTranslator } from '@jupyterlab/translation'; import { CommandRegistry } from '@lumino/commands'; -import { ReadonlyPartialJSONObject } from '@lumino/coreutils'; -import { Message } from '@lumino/messaging'; -import { ISignal, Signal } from '@lumino/signaling'; -import { CommandPalette } from '@lumino/widgets'; - -/** Structure of the JSON response from the server */ -interface IQuickOpenResponse { - readonly contents: { [key: string]: string[] }; - readonly scanSeconds: number; -} - -/** Makes a HTTP request for the server-side quick open scan */ -async function fetchContents( - path: string, - excludes: string[] -): Promise { - const query = excludes - .map(exclude => { - return 'excludes=' + encodeURIComponent(exclude); - }) - .join('&'); - - const settings = ServerConnection.makeSettings(); - const fullUrl = - URLExt.join(settings.baseUrl, 'jupyterlab-quickopen', 'api', 'files') + - '?' + - query + - '&path=' + - path; - const response = await ServerConnection.makeRequest( - fullUrl, - { method: 'GET' }, - settings - ); - if (response.status !== 200) { - throw new ServerConnection.ResponseError(response); - } - return await response.json(); -} - -/** - * Shows files nested under directories in the root notebooks directory configured on the server. - */ -class QuickOpenWidget extends CommandPalette { - private _pathSelected = new Signal(this); - private _settings: ReadonlyPartialJSONObject; - private _fileBrowser: FileBrowser; - - constructor( - defaultBrowser: IDefaultFileBrowser, - settings: ReadonlyPartialJSONObject, - options: CommandPalette.IOptions - ) { - super(options); - - this.id = 'jupyterlab-quickopen'; - this.title.iconClass = 'jp-SideBar-tabIcon jp-SearchIcon'; - this.title.caption = 'Quick Open'; - - this._settings = settings; - this._fileBrowser = defaultBrowser; - } - - /** Signal when a selected path is activated. */ - get pathSelected(): ISignal { - return this._pathSelected; - } - - /** Current extension settings */ - set settings(settings: ReadonlyPartialJSONObject) { - this._settings = settings; - } - - /** - * Refreshes the widget with the paths of files on the server. - */ - protected async onActivateRequest(msg: Message): Promise { - super.onActivateRequest(msg); - - // Fetch the current contents from the server - const path = this._settings.relativeSearch - ? this._fileBrowser.model.path - : ''; - const response = await fetchContents( - path, - this._settings.excludes as string[] - ); - - // Remove all paths from the view - this.clearItems(); - - for (const category in response.contents) { - for (const fn of response.contents[category]) { - // Creates commands that are relative file paths on the server - const command = `${category}/${fn}`; - if (!this.commands.hasCommand(command)) { - // Only add the command to the registry if it does not yet exist TODO: Track disposables - // and remove - this.commands.addCommand(command, { - label: fn, - execute: () => { - // Emit a selection signal - this._pathSelected.emit(command); - } - }); - } - // Make the file visible under its parent directory heading - this.addItem({ command, category }); - } - } - } -} +import { QuickOpenWidget } from './quickopen'; /** * Initialization data for the jupyterlab-quickopen extension. @@ -144,13 +32,14 @@ const extension: JupyterFrontEndPlugin = { const settings: ISettingRegistry.ISettings = await settingRegistry.load( extension.id ); - const widget: QuickOpenWidget = new QuickOpenWidget( - defaultFileBrowser, - settings.composite, - { - commands - } - ); + const widget: QuickOpenWidget = new QuickOpenWidget({ + defaultBrowser: defaultFileBrowser, + settings: settings.composite, + commandPaletteOptions: { commands }, + contents: app.serviceManager.contents, + // TODO: remove + useServer: false + }); // Listen for path selection signals and show the selected files in the appropriate // editor/viewer diff --git a/src/quickopen.ts b/src/quickopen.ts new file mode 100644 index 0000000..9c8b7f7 --- /dev/null +++ b/src/quickopen.ts @@ -0,0 +1,239 @@ +import { URLExt, PathExt } from '@jupyterlab/coreutils'; +import { FileBrowser, IDefaultFileBrowser } from '@jupyterlab/filebrowser'; +import { Contents, ServerConnection } from '@jupyterlab/services'; + +import { ReadonlyPartialJSONObject } from '@lumino/coreutils'; +import { Message } from '@lumino/messaging'; +import { ISignal, Signal } from '@lumino/signaling'; +import { CommandPalette } from '@lumino/widgets'; + +import { minimatch } from 'minimatch'; + +/** Structure of the JSON response from the server */ +interface IQuickOpenResponse { + readonly contents: { [key: string]: string[] }; + readonly scanSeconds: number; +} + +class DefaultDict extends Map { + private defaultFactory: () => V; + + constructor( + defaultFactory: () => V, + entries?: readonly (readonly [K, V])[] | null + ) { + super(entries); + this.defaultFactory = defaultFactory; + } + + get(key: K): V { + if (!this.has(key)) { + const defaultValue = this.defaultFactory(); + this.set(key, defaultValue); + return defaultValue; + } + return super.get(key)!; + } +} + +/** Makes a HTTP request for the server-side quick open scan */ +async function fetchServerContents( + path: string, + excludes: string[] +): Promise { + const query = excludes + .map(exclude => { + return 'excludes=' + encodeURIComponent(exclude); + }) + .join('&'); + + const settings = ServerConnection.makeSettings(); + const fullUrl = + URLExt.join(settings.baseUrl, 'jupyterlab-quickopen', 'api', 'files') + + '?' + + query + + '&path=' + + path; + const response = await ServerConnection.makeRequest( + fullUrl, + { method: 'GET' }, + settings + ); + if (response.status !== 200) { + throw new ServerConnection.ResponseError(response); + } + return await response.json(); +} + +async function fetchContents( + contents: Contents.IManager +): Promise { + const defaultIgnorePatterns = new Set([ + 'node_modules', + 'dist', + '.git', + '.cache', + 'build', + 'coverage', + 'tmp', + 'temp', + '*.log' + ]); + + // // Function to fetch the content of the .gitignore file + // const fetchGitignore = async (): Promise> => { + // try { + // const response = await contents.get('.gitignore', { content: true }); + // if (response.type === 'file') { + // const patterns = response.content + // .split('\n') + // .filter((line: string) => line.trim() !== '') + // .map((line: string) => line.replace(/\/$/, '')); + + // return new Set(patterns); + // } + // } catch (error) { + // console.warn('.gitignore file not found or could not be read.'); + // } + // return new Set(); + // }; + + const isIgnored = (path: string, ignorePatterns: Set): boolean => { + return [...ignorePatterns].some(pattern => { + return minimatch(path, pattern); + }); + }; + + // Function to fetch all file names, filtering out those matching .gitignore patterns + const dict = new DefaultDict(() => []); + const fetchAllFiles = async (path: string, ignorePatterns: Set) => { + try { + const response = await contents.get(path, { content: true }); + if (response.type === 'directory') { + for (const item of response.content) { + if (!isIgnored(item.path, ignorePatterns)) { + if (item.type === 'directory') { + await fetchAllFiles(item.path, ignorePatterns); + } else { + const basename = PathExt.basename(item.path); + const folders = item.path.split(basename)[0]; + dict.get(folders).push(basename); + } + } + } + } + } catch (error) { + console.error('Error fetching files:', error); + } + }; + + // Fetch .gitignore patterns + // const gitignorePatterns = await fetchGitignore(); + // console.log('.gitignore patterns:', gitignorePatterns); + + // Merge default ignore patterns with .gitignore patterns + // const combinedIgnorePatterns = new Set([ + // ...defaultIgnorePatterns, + // ...gitignorePatterns + // ]); + const combinedIgnorePatterns = new Set([...defaultIgnorePatterns]); + await fetchAllFiles('', combinedIgnorePatterns); + const results: { [key: string]: string[] } = {}; + for (const [key, value] of dict.entries()) { + results[key] = value; + } + return { + contents: results, + scanSeconds: 0 + }; +} + +/** + * Shows files nested under directories in the root notebooks directory configured on the server. + */ +export class QuickOpenWidget extends CommandPalette { + constructor(options: QuickOpenWidget.IOptions) { + super(options.commandPaletteOptions); + + this.id = 'jupyterlab-quickopen'; + this.title.iconClass = 'jp-SideBar-tabIcon jp-SearchIcon'; + this.title.caption = 'Quick Open'; + + const { defaultBrowser, settings } = options; + this._settings = settings; + this._fileBrowser = defaultBrowser; + this._contents = options.contents; + this._useServer = options.useServer ?? true; + } + + /** Signal when a selected path is activated. */ + get pathSelected(): ISignal { + return this._pathSelected; + } + + /** Current extension settings */ + set settings(settings: ReadonlyPartialJSONObject) { + this._settings = settings; + } + + /** + * Refreshes the widget with the paths of files on the server. + */ + protected async onActivateRequest(msg: Message): Promise { + super.onActivateRequest(msg); + + // Fetch the current contents from the server + const path = this._settings.relativeSearch + ? this._fileBrowser.model.path + : ''; + + let response: IQuickOpenResponse; + if (this._useServer) { + response = await fetchServerContents( + path, + this._settings.excludes as string[] + ); + } else { + response = await fetchContents(this._contents); + } + + // Remove all paths from the view + this.clearItems(); + + for (const category in response.contents) { + for (const fn of response.contents[category]) { + // Creates commands that are relative file paths on the server + const command = `${category}/${fn}`; + if (!this.commands.hasCommand(command)) { + // Only add the command to the registry if it does not yet exist TODO: Track disposables + // and remove + this.commands.addCommand(command, { + label: fn, + execute: () => { + // Emit a selection signal + this._pathSelected.emit(command); + } + }); + } + // Make the file visible under its parent directory heading + this.addItem({ command, category }); + } + } + } + + private _pathSelected = new Signal(this); + private _settings: ReadonlyPartialJSONObject; + private _fileBrowser: FileBrowser; + private _useServer = true; + private _contents: Contents.IManager; +} + +export namespace QuickOpenWidget { + export interface IOptions { + defaultBrowser: IDefaultFileBrowser; + settings: ReadonlyPartialJSONObject; + commandPaletteOptions: CommandPalette.IOptions; + contents: Contents.IManager; + useServer?: boolean; + } +} diff --git a/yarn.lock b/yarn.lock index 048eb5a..5dc4235 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3427,6 +3427,7 @@ __metadata: eslint: ^8.36.0 eslint-config-prettier: ^8.8.0 eslint-plugin-prettier: ^5.0.0 + minimatch: ^10.0.1 mkdirp: ^1.0.3 npm-run-all: ^4.1.5 prettier: ^3.0.0 @@ -3760,6 +3761,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^10.0.1": + version: 10.0.1 + resolution: "minimatch@npm:10.0.1" + dependencies: + brace-expansion: ^2.0.1 + checksum: f5b63c2f30606091a057c5f679b067f84a2cd0ffbd2dbc9143bda850afd353c7be81949ff11ae0c86988f07390eeca64efd7143ee05a0dab37f6c6b38a2ebb6c + languageName: node + linkType: hard + "minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2"