From f048d87f9bfadf29cdd8bd9f2eb7cf53d6019cfd Mon Sep 17 00:00:00 2001 From: Pacifique Linjanja Date: Mon, 13 Jan 2025 13:29:10 +0200 Subject: [PATCH 1/3] feat: add the support of URLs for the `nearImage()` method --- src/utils/base64.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/utils/base64.ts b/src/utils/base64.ts index 25991570..80c1612a 100644 --- a/src/utils/base64.ts +++ b/src/utils/base64.ts @@ -1,4 +1,5 @@ import fs from 'fs'; +import { httpClient } from '../connection/http.js'; const isFilePromise = (file: string | Buffer): Promise => new Promise((resolve, reject) => { @@ -22,6 +23,29 @@ const isFilePromise = (file: string | Buffer): Promise => }); }); +const isUrl = (file: string | Buffer): boolean => { + if (typeof file !== 'string') return false; + try { + const url = new URL(file); + return !!url; + } catch { + return false; + } +}; + +const downloadImageAsBase64 = async (url: string): Promise => { + try { + const client = httpClient({ + headers: { 'Content-Type': 'image/*' }, + host: '', + }); + const response = await client.externalGet(url); + return Buffer.from(response, 'binary').toString('base64'); + } catch (error) { + throw new Error(`Failed to download image from URL: ${url}`); + } +}; + const isBuffer = (file: string | Buffer): file is Buffer => file instanceof Buffer; const fileToBase64 = (file: string | Buffer): Promise => @@ -37,6 +61,8 @@ const fileToBase64 = (file: string | Buffer): Promise => }) : isBuffer(file) ? Promise.resolve(file.toString('base64')) + : isUrl(file) + ? downloadImageAsBase64(file) : Promise.resolve(file) ); @@ -44,7 +70,7 @@ const fileToBase64 = (file: string | Buffer): Promise => * This function converts a file buffer into a base64 string so that it can be * sent to Weaviate and stored as a media field. * - * @param {string | Buffer} file The media to convert either as a base64 string, a file path string, or as a buffer. If you passed a base64 string, the function does nothing and returns the string as is. + * @param {string | Buffer} file The media to convert either as a base64 string, a file path string, an url, or as a buffer. If you passed a base64 string, the function does nothing and returns the string as is. * @returns {string} The base64 string */ export const toBase64FromMedia = (media: string | Buffer): Promise => fileToBase64(media); From 833af4252ebc5c31c398a846295d5475f2299809 Mon Sep 17 00:00:00 2001 From: Pacifique Linjanja Date: Mon, 13 Jan 2025 13:56:56 +0200 Subject: [PATCH 2/3] tests: add unit tests for the `downloadImageFromURLAsBase64()` function --- src/utils/base64.test.ts | 64 ++++++++++++++++++++++++++++++++++++++++ src/utils/base64.ts | 16 ++++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 src/utils/base64.test.ts diff --git a/src/utils/base64.test.ts b/src/utils/base64.test.ts new file mode 100644 index 00000000..bb33d3af --- /dev/null +++ b/src/utils/base64.test.ts @@ -0,0 +1,64 @@ +import { httpClient } from '../connection/http.js'; +import { downloadImageFromURLAsBase64 } from './base64.js'; + +jest.mock('../connection/http.js'); + +describe('downloadImageFromURLAsBase64()', () => { + const mockHttpClient = { + externalGet: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + (httpClient as jest.Mock).mockReturnValue(mockHttpClient); + }); + + it('should convert a downloaded image to base64', async () => { + const mockUrl = 'https://example.com/image.jpg'; + const mockImageData = Buffer.from('image binary data'); + + mockHttpClient.externalGet.mockResolvedValue(mockImageData); + + const result = await downloadImageFromURLAsBase64(mockUrl); + + expect(result).toBe(mockImageData.toString('base64')); + expect(httpClient).toHaveBeenCalledWith({ headers: { 'Content-Type': 'image/*' }, host: '' }); + expect(mockHttpClient.externalGet).toHaveBeenCalledWith(mockUrl); + }); + + it('should throw an error if the URL is invalid', async () => { + const invalidUrl = 'invalid-url'; + + await expect(downloadImageFromURLAsBase64(invalidUrl)).rejects.toThrow('Invalid URL'); + }); + + it('should throw an error if the image download fails', async () => { + const mockUrl = 'https://example.com/image.jpg'; + + mockHttpClient.externalGet.mockRejectedValue(new Error('Network error')); + + await expect(downloadImageFromURLAsBase64(mockUrl)).rejects.toThrow('Failed to download image from URL'); + expect(httpClient).toHaveBeenCalledWith({ headers: { 'Content-Type': 'image/*' }, host: '' }); + expect(mockHttpClient.externalGet).toHaveBeenCalledWith(mockUrl); + }); + + it('should handle empty response data gracefully', async () => { + const mockUrl = 'https://example.com/image.jpg'; + + mockHttpClient.externalGet.mockResolvedValue(Buffer.alloc(0)); + + const result = await downloadImageFromURLAsBase64(mockUrl); + + expect(result).toBe(''); + expect(httpClient).toHaveBeenCalledWith({ headers: { 'Content-Type': 'image/*' }, host: '' }); + expect(mockHttpClient.externalGet).toHaveBeenCalledWith(mockUrl); + }); + + it('should throw an error if the response is not a buffer', async () => { + const mockUrl = 'wrong-url.com'; + + mockHttpClient.externalGet.mockResolvedValue('not a buffer'); + + await expect(downloadImageFromURLAsBase64(mockUrl)).rejects.toThrow('Invalid URL'); + }); +}); diff --git a/src/utils/base64.ts b/src/utils/base64.ts index 80c1612a..408df3cc 100644 --- a/src/utils/base64.ts +++ b/src/utils/base64.ts @@ -33,14 +33,24 @@ const isUrl = (file: string | Buffer): boolean => { } }; -const downloadImageAsBase64 = async (url: string): Promise => { +export const downloadImageFromURLAsBase64 = async (url: string): Promise => { + if (!isUrl(url)) { + throw new Error('Invalid URL'); + } + try { const client = httpClient({ headers: { 'Content-Type': 'image/*' }, host: '', }); + const response = await client.externalGet(url); - return Buffer.from(response, 'binary').toString('base64'); + + if (!Buffer.isBuffer(response)) { + throw new Error('Response is not a buffer'); + } + + return response.toString('base64'); } catch (error) { throw new Error(`Failed to download image from URL: ${url}`); } @@ -62,7 +72,7 @@ const fileToBase64 = (file: string | Buffer): Promise => : isBuffer(file) ? Promise.resolve(file.toString('base64')) : isUrl(file) - ? downloadImageAsBase64(file) + ? downloadImageFromURLAsBase64(file) : Promise.resolve(file) ); From df5e1572281319fbd58dd1aba41c7ca464cdf12a Mon Sep 17 00:00:00 2001 From: Pacifique Linjanja Date: Mon, 13 Jan 2025 17:55:46 +0200 Subject: [PATCH 3/3] chore: minor update in the `isUrl()` utility types --- src/utils/base64.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/base64.ts b/src/utils/base64.ts index 408df3cc..57a09ef8 100644 --- a/src/utils/base64.ts +++ b/src/utils/base64.ts @@ -23,7 +23,7 @@ const isFilePromise = (file: string | Buffer): Promise => }); }); -const isUrl = (file: string | Buffer): boolean => { +const isUrl = (file: string): file is string => { if (typeof file !== 'string') return false; try { const url = new URL(file);