From c58a00d33f9e596b648f589b44e575764e4d1ecb Mon Sep 17 00:00:00 2001 From: minottic Date: Tue, 23 Apr 2024 15:17:12 +0200 Subject: [PATCH] Use search with deepBy arg instead of inferring --- .../basesnippet.controller.acceptance.ts | 30 +-- .../acceptance/file.controller.acceptance.ts | 8 +- .../location.controller.acceptance.ts | 8 +- .../logbook.controller.acceptance.ts | 8 +- .../paragraph.controller.acceptance.ts | 8 +- .../acceptance/task.controller.acceptance.ts | 8 +- .../src/__tests__/unit/utils.misc.unit.ts | 46 +++++ .../src/controllers/basesnippet.controller.ts | 2 + .../mixins/basesnippet.repository-mixin.ts | 186 +++++++++--------- sci-log-db/src/utils/misc.ts | 13 ++ 10 files changed, 175 insertions(+), 142 deletions(-) diff --git a/sci-log-db/src/__tests__/acceptance/basesnippet.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/basesnippet.controller.acceptance.ts index ec3e75c9..0c81d153 100644 --- a/sci-log-db/src/__tests__/acceptance/basesnippet.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/basesnippet.controller.acceptance.ts @@ -240,13 +240,9 @@ describe('Basesnippet', function (this: Suite) { }); it('Search with token should return 200 and matching body.tags', async () => { - const includeTags = {include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/basesnippets/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/basesnippets/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) @@ -262,12 +258,12 @@ describe('Basesnippet', function (this: Suite) { }); it('Search with token should return 200 and matching body.tags + body.textcontent', async () => { - const includeTags = {include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client .get( - `/basesnippets/search=aSearchable Tex ${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, + `/basesnippets/search=aSearchable Tex?filter=${JSON.stringify( + includeTags, + )}`, ) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') @@ -284,8 +280,9 @@ describe('Basesnippet', function (this: Suite) { }); it('Search with token should return 200 and matching readACL', async () => { + const filter = {where: {readACL: 'basesnippetAcceptance'}}; await client - .get(`/basesnippets/search=@basesnippetAcceptance`) + .get(`/basesnippets/search=%00?filter=${JSON.stringify(filter)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) @@ -329,7 +326,10 @@ describe('Basesnippet', function (this: Suite) { }); it('Search with token should return 200 and matching subsnippets', async () => { - const includeTags = {include: ['subsnippets']}; + const includeTags = { + where: {tags: 'aSubsnippetTag'}, + include: ['subsnippets'], + }; await client .post('/basesnippets') .set('Authorization', 'Bearer ' + token) @@ -342,9 +342,9 @@ describe('Basesnippet', function (this: Suite) { await client .get( - `/basesnippets/search=${encodeURIComponent( - '#aSubsnippetTag', - )}?filter=${JSON.stringify(includeTags)}`, + `/basesnippets/search=aSearch?filter=${JSON.stringify( + includeTags, + )}&deepby=tags`, ) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') diff --git a/sci-log-db/src/__tests__/acceptance/file.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/file.controller.acceptance.ts index 77e0c6dc..d25975d6 100644 --- a/sci-log-db/src/__tests__/acceptance/file.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/file.controller.acceptance.ts @@ -187,13 +187,9 @@ describe('File controller services', function (this: Suite) { }); it('Search index with token should return 200 and matching body.tags', async () => { - const includeTags = {fields: {tags: true}, include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/filesnippet/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/filesnippet/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) diff --git a/sci-log-db/src/__tests__/acceptance/location.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/location.controller.acceptance.ts index b9cc4e10..e4c9faec 100644 --- a/sci-log-db/src/__tests__/acceptance/location.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/location.controller.acceptance.ts @@ -191,13 +191,9 @@ describe('Location', function (this: Suite) { }); it('Search index with token should return 200 and matching body.tags', async () => { - const includeTags = {fields: {tags: true}, include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/locations/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/locations/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) diff --git a/sci-log-db/src/__tests__/acceptance/logbook.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/logbook.controller.acceptance.ts index b675297b..7fef3c85 100644 --- a/sci-log-db/src/__tests__/acceptance/logbook.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/logbook.controller.acceptance.ts @@ -184,13 +184,9 @@ describe('Logbook', function (this: Suite) { }); it('Search index with token should return 200 and matching body.tags', async () => { - const includeTags = {fields: {tags: true}, include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/logbooks/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/logbooks/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) diff --git a/sci-log-db/src/__tests__/acceptance/paragraph.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/paragraph.controller.acceptance.ts index c66c202a..1b751c28 100644 --- a/sci-log-db/src/__tests__/acceptance/paragraph.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/paragraph.controller.acceptance.ts @@ -165,13 +165,9 @@ describe('Paragraph', function (this: Suite) { }); it('Search index with token should return 200 and matching body.tags', async () => { - const includeTags = {fields: {tags: true}, include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/paragraphs/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/paragraphs/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) diff --git a/sci-log-db/src/__tests__/acceptance/task.controller.acceptance.ts b/sci-log-db/src/__tests__/acceptance/task.controller.acceptance.ts index f70723b3..1b3fdd12 100644 --- a/sci-log-db/src/__tests__/acceptance/task.controller.acceptance.ts +++ b/sci-log-db/src/__tests__/acceptance/task.controller.acceptance.ts @@ -162,13 +162,9 @@ describe('TaskRepositorySnippet', function (this: Suite) { }); it('Search index with token should return 200 and matching body.tags', async () => { - const includeTags = {fields: {tags: true}, include: ['subsnippets']}; + const includeTags = {where: {tags: 'aSearchableTag'}}; await client - .get( - `/tasks/search=${encodeURIComponent( - '#aSearchableTag', - )}?filter=${JSON.stringify(includeTags)}`, - ) + .get(`/tasks/search=%00?filter=${JSON.stringify(includeTags)}`) .set('Authorization', 'Bearer ' + token) .set('Content-Type', 'application/json') .expect(200) diff --git a/sci-log-db/src/__tests__/unit/utils.misc.unit.ts b/sci-log-db/src/__tests__/unit/utils.misc.unit.ts index b3473736..14bd767b 100644 --- a/sci-log-db/src/__tests__/unit/utils.misc.unit.ts +++ b/sci-log-db/src/__tests__/unit/utils.misc.unit.ts @@ -13,7 +13,9 @@ import { standardiseIncludes, sanitizeTextContent, sanitizeTextContentInPlace, + removeDeepBy, } from '../../utils/misc'; +import {Where} from '@loopback/repository'; describe('Utils unit tests', function (this: Suite) { it('Should filterEmptySubsnippets', () => { @@ -316,4 +318,48 @@ describe('Utils unit tests', function (this: Suite) { expect(t.input).to.be.eql(t.expected); }); }); + + [ + {input: [{key1: 'a', key2: 'b'}, ['key2']], expected: {key1: 'a'}}, + {input: [{a: {key1: 'a'}, key2: 'b'}, ['key1', 'key2']], expected: {a: {}}}, + { + input: [{a: {key1: 'a'}, key2: 'b'}, ['key1']], + expected: {a: {}, key2: 'b'}, + }, + { + input: [{a: {b: {key1: 'a'}}, key2: 'b'}, ['key1']], + expected: {a: {b: {}}, key2: 'b'}, + }, + ].forEach((t, i) => { + it(`Should test removeDeepBy ${i}`, () => { + removeDeepBy(t.input[0] as Where, (key: string) => + (t.input[1] as string[]).includes(key), + ); + expect(t.input[0]).to.be.eql(t.expected); + }); + }); + + [ + {input: [{key1: 'a', key2: 'b'}, ['key2']], expected: {key2: 'b'}}, + { + input: [{a: {key1: 'a'}, key2: 'b'}, ['a', 'key2']], + expected: {a: {}, key2: 'b'}, + }, + { + input: [{a: {key1: 'a'}, key2: 'b'}, ['key1']], + expected: {}, + }, + { + input: [{a: {b: {key1: 'a'}}, key2: 'b'}, ['key2']], + expected: {key2: 'b'}, + }, + ].forEach((t, i) => { + it(`Should test removeDeepBy with negation ${i}`, () => { + removeDeepBy( + t.input[0] as Where, + (key: string) => !(t.input[1] as string[]).includes(key), + ); + expect(t.input[0]).to.be.eql(t.expected); + }); + }); }); diff --git a/sci-log-db/src/controllers/basesnippet.controller.ts b/sci-log-db/src/controllers/basesnippet.controller.ts index 0187cf80..3f0fcb00 100644 --- a/sci-log-db/src/controllers/basesnippet.controller.ts +++ b/sci-log-db/src/controllers/basesnippet.controller.ts @@ -176,11 +176,13 @@ export class BasesnippetController { @param.path.string('search') search: string, @param.filter(Basesnippet, {exclude: 'fields'}) filter?: Filter, + @param.array('deepby', 'query', {type: 'string'}) deepBy?: string[], ): Promise { const snippetsFiltered = await this.basesnippetRepository.findWithSearch( search, this.user, filter, + deepBy, ); return snippetsFiltered; } diff --git a/sci-log-db/src/mixins/basesnippet.repository-mixin.ts b/sci-log-db/src/mixins/basesnippet.repository-mixin.ts index 7d44d808..37e1d7eb 100644 --- a/sci-log-db/src/mixins/basesnippet.repository-mixin.ts +++ b/sci-log-db/src/mixins/basesnippet.repository-mixin.ts @@ -21,6 +21,7 @@ import { arrayOfUniqueFrom, concatOwnerAccessGroups, filterEmptySubsnippets, + removeDeepBy, standardiseIncludes, } from '../utils/misc'; import {AutoAddRepository} from '../repositories/autoadd.repository.base'; @@ -262,35 +263,71 @@ function FindWithSearchRepositoryMixin< Relations extends {subsnippets: M}, R extends MixinTarget>, >(superClass: R) { + type DeepBy = keyof M; class Mixed extends superClass { user: UserProfile; + private _deepBy: DeepBy[] = []; + private _filter: Filter; + private _searchCondition: Condition; + private _deepCondition: Where; + + private extractDeepCondition(where: Where | undefined) { + if (!where) return; + this._deepCondition = JSON.parse(JSON.stringify(where)); + removeDeepBy( + this._deepCondition, + (key: string) => + ![ + ...this._deepBy, + ...Object.getOwnPropertyNames(WhereBuilder.prototype), + ].includes(key), + ); + } async recursivelyApplySearchCondition( - fullFilter: Filter | undefined, - condition: Condition, filter: Filter | undefined, snippets: {[id: string]: Basesnippet} = {}, - depth = 0, + depth = 1, ) { const whereCopy = JSON.parse(JSON.stringify(filter?.where ?? {})); - this.addSearchToFilter(filter, condition); - const snippet = await this.find(fullFilter, {currentUser: this.user}); - const fakeFirstLevel = {subsnippets: snippet} as unknown as Basesnippet; - depth += 1; - filterEmptySubsnippets(fakeFirstLevel, depth); - fakeFirstLevel.subsnippets?.map(snip => { - if (!_.has(snippets, snip.id)) snippets[snip.id] = snip; - }); - if (filter) filter.where = whereCopy; + this.enrichFilter(filter); + const snippet = await this.find(this._filter, {currentUser: this.user}); + this.removeEmptySubsnippets(snippet, depth, snippets); + this.resetWhere(filter, whereCopy); if (!filter?.include || filter.include.length === 0) return snippets; for (const relation of filter?.include as Inclusion[]) { relation.scope = relation.scope ?? {}; await this.recursivelyApplySearchCondition( - fullFilter, - condition, relation.scope as Filter, snippets, - depth, + depth + 1, + ); + } + } + + private enrichFilter(filter: Filter | undefined) { + this.addSearchToFilter(filter); + this.addDeepToFilter(filter); + } + + private removeEmptySubsnippets( + snippet: (M & Relations)[], + depth: number, + snippets: {[id: string]: Basesnippet}, + ) { + const fakeFirstLevel = {subsnippets: snippet} as unknown as Basesnippet; + filterEmptySubsnippets(fakeFirstLevel, depth); + fakeFirstLevel.subsnippets?.map(snip => { + if (!_.has(snippets, snip.id)) snippets[snip.id] = snip; + }); + } + + private resetWhere(filter: Filter | undefined, whereCopy: Where) { + if (filter) { + filter.where = whereCopy; + this.extractDeepCondition(filter.where); + removeDeepBy(filter.where as Where, (key: DeepBy) => + this._deepBy.includes(key), ); } } @@ -299,18 +336,11 @@ function FindWithSearchRepositoryMixin< search: string, user: UserProfile, filter: Filter = {}, + deepBy: DeepBy[] = [], ): Promise { - delete filter?.fields; - standardiseIncludes(filter); - const searchCondition = this.buildAdditionalConditions(search); - this.user = user; + this.init(filter, search, user, deepBy); const snippets: {[id: string]: Basesnippet} = {}; - await this.recursivelyApplySearchCondition( - filter, - searchCondition, - filter, - snippets, - ); + await this.recursivelyApplySearchCondition(filter, snippets); const desc = filter.order?.[0] === 'defaultOrder DESC'; return Object.values(snippets).sort((first, second) => desc @@ -319,87 +349,49 @@ function FindWithSearchRepositoryMixin< ); } - private addSearchToFilter( - filter: Filter | undefined, - additionalConditions: Condition, + private init( + filter: Filter, + search: string, + user: UserProfile, + deepBy: (keyof M)[], ) { - if (!_.isEmpty(additionalConditions)) - this._addSearchCondition(filter, additionalConditions); + delete filter?.fields; + standardiseIncludes(filter); + this._filter = filter; + this._searchCondition = this.buildSearchCondition(search); + this.user = user; + this._deepBy = deepBy; } - private _addSearchCondition( - filter: Filter | undefined, - includeOrs: Where, - ) { - const searchCondition = new WhereBuilder(filter?.where) - .and(includeOrs) - .build(); - return new FilterBuilder(filter).where(searchCondition).build(); + private addSearchToFilter(filter: Filter | undefined) { + this._addCondition(filter, this._searchCondition); } - private _additionalConditionBuilder(commonSearchableFields: Condition) { - return { - and: Object.entries(commonSearchableFields).flatMap(([k, v]) => { - return {[k]: v} as Where; - }), - } as Condition; + private addDeepToFilter(filter: Filter | undefined) { + this._addCondition(filter, this._deepCondition); } - private buildAdditionalConditions(search: string) { - const additionalConditions: { - tags?: {inq: string[]}; - readACL?: {inq: string[]}; - or?: { - textcontent?: {regexp: RegExp}; - name?: {regexp: RegExp}; - description?: {regexp: RegExp}; - }[]; - } & Condition = {}; - let searchText = ''; - search.split(' ').map(searchTerms => { - if (searchTerms.startsWith('#')) { - additionalConditions.tags = additionalConditions.tags ?? {inq: []}; - additionalConditions.tags.inq.push(searchTerms.slice(1)); - } else if (searchTerms.startsWith('@')) { - additionalConditions.readACL = additionalConditions.readACL ?? { - inq: [], - }; - additionalConditions.readACL.inq.push(searchTerms.slice(1)); - } else searchText += ` ${searchTerms}`; - }); - this.addSearchOr(searchText, additionalConditions); - if ( - (additionalConditions.or && - Object.keys(additionalConditions).length > 2) || - Object.keys(additionalConditions).length > 1 - ) - return this._additionalConditionBuilder(additionalConditions); - return additionalConditions; + private _addCondition( + filter: Filter | undefined, + additionalWhere: Where, + ) { + if (_.isEmpty(additionalWhere)) return; + let additionalCondition = additionalWhere; + if (filter?.where) + additionalCondition = {and: [additionalWhere, filter.where]}; + return new FilterBuilder(filter).where(additionalCondition).build(); } - private addSearchOr( - searchText: string, - additionalConditions: { - tags?: {inq: string[]} | undefined; - readACL?: {inq: string[]} | undefined; - or?: - | { - textcontent?: {regexp: RegExp} | undefined; - name?: {regexp: RegExp} | undefined; - description?: {regexp: RegExp} | undefined; - }[] - | undefined; - } & Condition, - ) { - if (!searchText) return; - searchText = searchText.trimStart(); - const searchRegex = {regexp: new RegExp(`.*${searchText}.*`, 'i')}; - const searchCondition = [ - {name: searchRegex}, - {description: searchRegex}, - {htmlTextcontent: searchRegex}, - ]; - additionalConditions.or = searchCondition; + private buildSearchCondition(search: string) { + if (!search || encodeURIComponent(search) === '%00') return {}; + const searchRegex = {regexp: new RegExp(`.*${search}.*`, 'i')}; + return { + or: [ + {name: searchRegex}, + {description: searchRegex}, + {htmlTextcontent: searchRegex}, + ], + } as Condition; } async findIndexInBuffer( diff --git a/sci-log-db/src/utils/misc.ts b/sci-log-db/src/utils/misc.ts index 53e69834..1773c055 100644 --- a/sci-log-db/src/utils/misc.ts +++ b/sci-log-db/src/utils/misc.ts @@ -4,6 +4,7 @@ import { JsonSchema, ModelDefinition, Filter, + Where, } from '@loopback/repository'; import {JsonSchemaOptions} from '@loopback/repository-json-schema'; import { @@ -233,3 +234,15 @@ export function sanitizeTextContentInPlace(data?: { if (!sanitisedText) return; data.htmlTextcontent = sanitisedText; } + +export function removeDeepBy( + where: Where, + condition: Function, +) { + if (!where || typeof where !== 'object') return; + const isObject = _.isPlainObject(where); + Object.entries(where).map(([key, value]) => { + if (condition(key) && isObject) delete where[key as keyof Where]; + else removeDeepBy(value as Where, condition); + }); +}