Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hooks, improvements, and hacks to puter.com, and implementation of a puter integration #39

Merged
merged 14 commits into from
Jan 31, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Support importing Excel files; choose the right parser, and support b…
…inary data. Also a workaround for puter read() bug
dsagal committed Jan 26, 2025
commit 393431587f6010ce398e9273179c20441ed23726
8 changes: 5 additions & 3 deletions ext/app/pipe/GristOverrides.ts
Original file line number Diff line number Diff line change
@@ -14,11 +14,13 @@ export interface GristOverrides {
*/
seedFile?: string|URL|Uint8Array;
/**
* .csv file URL.
* URL of a file to fetch for importing, or a File object to import directly.
* The extension in the name (or in the URL path) determines how it's parsed.
*/
initialData?: string;
initialData?: string|File;
/**
* .csv file content to load. Not used when initialData is set.
* .csv file content to load. Not used when initialData is set. Note that this is ONLY for csv
* data. For other filetypes (e.g. xlsx), set initialData to a File with a suitable name.
*/
initialContent?: string;
fakeUrl?: string;
3 changes: 2 additions & 1 deletion ext/app/pipe/webworker.js
Original file line number Diff line number Diff line change
@@ -42,7 +42,8 @@ sandbox = sandbox_mod.default_sandbox = sandbox_mod.Sandbox(None, None)
sandbox.run = lambda: print("Sandbox is running")

def save_file(path, content):
with open(path, 'w') as f:
mode = 'w' if isinstance(content, str) else 'wb'
with open(path, mode) as f:
f.write(content)

sandbox.register('save_file', save_file)
41 changes: 31 additions & 10 deletions ext/app/server/lib/CommStub.ts
Original file line number Diff line number Diff line change
@@ -75,11 +75,11 @@ export class Comm extends dispose.Disposable implements GristServerAPI, DocList
(window as any).gristActiveDoc = this.ad;
//await this.ad.createEmptyDoc({});
const hasSeed = gristOverrides.seedFile;
const initialDataUrl = gristOverrides.initialData;
const initialData = gristOverrides.initialData;
const initialContent = gristOverrides.initialContent;
await this.ad.loadDoc({mode: 'system'}, {
forceNew: !hasSeed,
skipInitialTable: hasSeed || initialDataUrl || initialContent,
skipInitialTable: hasSeed || initialData || initialContent,
useExisting: true,
});
this.client = {
@@ -151,8 +151,8 @@ export class Comm extends dispose.Disposable implements GristServerAPI, DocList
gristOverrides.expressApp = this.expressApp;
if (initialContent) {
await this._loadInitialContent(initialContent);
} else if (initialDataUrl) {
await this._loadInitialData(initialDataUrl);
} else if (initialData) {
await this._loadInitialData(initialData);
}

return {
@@ -184,7 +184,7 @@ export class Comm extends dispose.Disposable implements GristServerAPI, DocList
return window.location.href;
}

private async _loadInitialData(initialDataUrl: string) {
private async _readFromURL(initialDataUrl: string): Promise<File> {
// If we are in a iframe, we need to use the parent window to fetch the data.
// This is hack to fix a bug in FF https://bugzilla.mozilla.org/show_bug.cgi?id=1741489, and shouldn't
// affect other browsers.
@@ -195,18 +195,39 @@ export class Comm extends dispose.Disposable implements GristServerAPI, DocList
if (!response.ok) {
throw new Error(`Failed to load initial data from ${initialDataUrl}: ${response.statusText}`);
}
const content = await response.text();
const content = await response.blob();
// Extract filename from end of URL
const originalFilename = initialDataUrl.match(/[^/]+$/)?.[0] || "data.csv";
await this._loadInitialContent(content, originalFilename);
return new File([content], originalFilename);
}

private async _loadInitialContent(content: string, originalFilename: string = "data.csv") {
const path = "/tmp/data.csv";
private async _loadInitialData(initialData: string|File) {
if (typeof initialData === 'string') {
initialData = await this._readFromURL(initialData);
}
const content = new Uint8Array(await initialData.arrayBuffer());
await this._loadInitialContent(content, initialData.name);
}

private async _loadInitialContent(content: string|Uint8Array, originalFilename: string = "data.csv") {
// Corresponds to core/plugins/core/manifest.yml.
const fileParsers = {
csv_parser: ['csv', 'tsv', 'dsv', 'txt'],
xls_parser: ['xlsx', 'xlsm'],
json_parser: ['json'],
};
// Turn into a map of 'csv' -> 'csv_parser', etc.
const parserMap = new Map(Object.entries(fileParsers).flatMap(([parser, lst]) => lst.map(ext => [ext, parser])));

const basename = originalFilename.split('/').pop()!;
const extension = basename.split('.').pop()!;
const parserName = parserMap.get(extension);
if (!parserName) { throw new Error("File format is not supported"); }
const path = `/tmp/${basename}`;
const parseOptions = {};
await this.ad._pyCall("save_file", path, content);
const parsedFile = await this.ad._pyCall(
"csv_parser.parseFile",
`${parserName}.parseFile`,
{path, origName: originalFilename},
parseOptions,
);
38 changes: 33 additions & 5 deletions page/puter.js
Original file line number Diff line number Diff line change
@@ -78,19 +78,47 @@ const behaviorOverrides = {
rename,
};


// XXX There is a bug on puter where some files are not readable. But they become readable if you
// open them at least once via showOpenFilePicker(). So do that so that it's at least somewhat
// possible to open.
async function readItemHack(item) {
try {
return await item.read();
} catch (e) {
if (e.code !== 'subject_does_not_exist') {
throw e;
}
const answer = await puter.ui.alert(`Failed to open: ${e.message}`, [
{label: 'Open another way', value: 'open', type: 'primary'},
{label: 'Cancel', value: 'cancel'},
]);
if (answer !== 'open') {
puter.exit();
return;
}
}
item = await puter.ui.showOpenFilePicker();
return await item.read();
}

const supportedExtensions = ['.csv', '.xlsx', '.tsv', '.dsv', '.txt', '.xlsm', '.json'];

async function openGristWithItem(item) {
try {
const config = {name: 'Untitled document', behaviorOverrides};
if (item) {
const {name, ext} = getNameFromFSItem(item);
config.name = name;
if (['.csv', '.xlsx', '.tsv'].includes(ext.toLowerCase())) {
// initialContent is used for imports.
config.initialContent = await (await item.read()).text();
} else if (ext.toLowerCase() === '.grist') {
if (ext.toLowerCase() === '.grist') {
// initialFile is used for opening existing Grist docs.
config.initialFile = await (await item.read()).bytes();
config.initialFile = new Uint8Array(await (await item.read()).arrayBuffer());
setCurrentPuterFSItem(item, true);
} else if (supportedExtensions.includes(ext.toLowerCase())) {
// initialData is used for imports.
const content = await readItemHack(item);
const nameWithExt = name + ext.toLowerCase();
config.initialData = new File([content], nameWithExt);
} else {
throw new Error("Unrecognized file type");
}